You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/03/15 16:32:16 UTC
[01/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 a5dfae69f -> ce23d9adf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
deleted file mode 100644
index 8cb57d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
-
-import org.apache.hadoop.cblock.meta.ContainerDescriptor;
-import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * NOTE : This class is only for testing purpose.
- *
- * Mock an underlying container storage layer, expose to CBlock to perform
- * IO. While in this mock implementation, a container is nothing more than
- * a in memory hashmap.
- *
- * This is to allow volume creation call and perform standalone tests.
- */
-public final class ContainerLookUpService {
- private static ConcurrentHashMap<String, ContainerDescriptor>
- containers = new ConcurrentHashMap<>();
-
- /**
- * Return an *existing* container with given Id.
- *
- * TODO : for testing purpose, return a new container if the given Id
- * is not found
- *
- * found
- * @param containerID
- * @return the corresponding pipeline instance (create if not exist)
- */
- public static ContainerDescriptor lookUp(String containerID)
- throws IOException {
- if (!containers.containsKey(containerID)) {
- Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
- containerID);
- ContainerDescriptor cd = new ContainerDescriptor(containerID);
- cd.setPipeline(pipeline);
- containers.put(containerID, cd);
- }
- return containers.get(containerID);
- }
-
- public static void addContainer(String containerID) throws IOException {
- Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
- containerID);
- ContainerDescriptor cd = new ContainerDescriptor(containerID);
- cd.setPipeline(pipeline);
- containers.put(containerID, cd);
- }
-
- private ContainerLookUpService() {
-
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
deleted file mode 100644
index 4aced57..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-nvd3-1.0.9.min.js
+++ /dev/null
@@ -1 +0,0 @@
-!function(window){"use strict";var nv=window.nv;"undefined"!=typeof exports&&(nv=require("nvd3")),angular.module("nvd3",[]).directive("nvd3",["nvd3Utils",function(nvd3Utils){return{restrict:"AE",scope:{data:"=",options:"=",api:"=?",events:"=?",config:"=?",onReady:"&?"},link:function(scope,element,attrs){function configure(chart,options,chartType){chart&&options&&angular.forEach(chart,function(value,key){"_"===key[0]||("dispatch"===key?(void 0!==options[key]&&null!==options[key]||scope._config.extended&&(options[key]={}),configureEvents(value,options[key])):"tooltip"===key?(void 0!==options[key]&&null!==options[key]||scope._config.extended&&(options[key]={}),configure(chart[key],options[key],chartType)):"contentGenerator"===key?options[key]&&chart[key](options[key]):-1===["axis","clearHighlights","defined","highlightPoint","nvPointerEventsClass","options","rangeBand","rangeBands","scatter","open","close","node"].indexOf(key)&&(void 0===options[key]||null===options[key]?scope._config.
extended&&(options[key]=value()):chart[key](options[key])))})}function configureEvents(dispatch,options){dispatch&&options&&angular.forEach(dispatch,function(value,key){void 0===options[key]||null===options[key]?scope._config.extended&&(options[key]=value.on):dispatch.on(key+"._",options[key])})}function configureWrapper(name){var _=nvd3Utils.deepExtend(defaultWrapper(name),scope.options[name]||{});scope._config.extended&&(scope.options[name]=_);var wrapElement=angular.element("<div></div>").html(_.html||"").addClass(name).addClass(_.className).removeAttr("style").css(_.css);_.html||wrapElement.text(_.text),_.enable&&("title"===name?element.prepend(wrapElement):"subtitle"===name?angular.element(element[0].querySelector(".title")).after(wrapElement):"caption"===name&&element.append(wrapElement))}function configureStyles(){var _=nvd3Utils.deepExtend(defaultStyles(),scope.options.styles||{});scope._config.extended&&(scope.options.styles=_),angular.forEach(_.classes,function(value,key){
value?element.addClass(key):element.removeClass(key)}),element.removeAttr("style").css(_.css)}function defaultWrapper(_){switch(_){case"title":return{enable:!1,text:"Write Your Title",className:"h4",css:{width:scope.options.chart.width+"px",textAlign:"center"}};case"subtitle":return{enable:!1,text:"Write Your Subtitle",css:{width:scope.options.chart.width+"px",textAlign:"center"}};case"caption":return{enable:!1,text:"Figure 1. Write Your Caption text.",css:{width:scope.options.chart.width+"px",textAlign:"center"}}}}function defaultStyles(){return{classes:{"with-3d-shadow":!0,"with-transitions":!0,gallery:!1},css:{}}}function dataWatchFn(newData,oldData){newData!==oldData&&(scope._config.disabled||(scope._config.refreshDataOnly?scope.api.update():scope.api.refresh()))}var defaultConfig={extended:!1,visible:!0,disabled:!1,refreshDataOnly:!0,deepWatchOptions:!0,deepWatchData:!0,deepWatchDataDepth:2,debounce:10,debounceImmediate:!0};scope.isReady=!1,scope._config=angular.extend(defaultC
onfig,scope.config),scope.api={refresh:function(){scope.api.updateWithOptions(),scope.isReady=!0},refreshWithTimeout:function(t){setTimeout(function(){scope.api.refresh()},t)},update:function(){scope.chart&&scope.svg?"sunburstChart"===scope.options.chart.type?scope.svg.datum(angular.copy(scope.data)).call(scope.chart):scope.svg.datum(scope.data).call(scope.chart):scope.api.refresh()},updateWithTimeout:function(t){setTimeout(function(){scope.api.update()},t)},updateWithOptions:function(options){if(arguments.length){if(scope.options=options,scope._config.deepWatchOptions&&!scope._config.disabled)return}else options=scope.options;scope.api.clearElement(),angular.isDefined(options)!==!1&&scope._config.visible&&(scope.chart=nv.models[options.chart.type](),scope.chart.id=Math.random().toString(36).substr(2,15),angular.forEach(scope.chart,function(value,key){"_"===key[0]||["clearHighlights","highlightPoint","id","options","resizeHandler","state","open","close","tooltipContent"].indexOf(key
)>=0||("dispatch"===key?(void 0!==options.chart[key]&&null!==options.chart[key]||scope._config.extended&&(options.chart[key]={}),configureEvents(scope.chart[key],options.chart[key])):["bars","bars1","bars2","boxplot","bullet","controls","discretebar","distX","distY","focus","interactiveLayer","legend","lines","lines1","lines2","multibar","pie","scatter","scatters1","scatters2","sparkline","stack1","stack2","sunburst","tooltip","x2Axis","xAxis","y1Axis","y2Axis","y3Axis","y4Axis","yAxis","yAxis1","yAxis2"].indexOf(key)>=0||"stacked"===key&&"stackedAreaChart"===options.chart.type?(void 0!==options.chart[key]&&null!==options.chart[key]||scope._config.extended&&(options.chart[key]={}),configure(scope.chart[key],options.chart[key],options.chart.type)):"focusHeight"===key&&"lineChart"===options.chart.type||"focusHeight"===key&&"lineWithFocusChart"===options.chart.type||("xTickFormat"!==key&&"yTickFormat"!==key||"lineWithFocusChart"!==options.chart.type)&&("tooltips"===key&&"boxPlotChart"=
==options.chart.type||("tooltipXContent"!==key&&"tooltipYContent"!==key||"scatterChart"!==options.chart.type)&&("x"!==key&&"y"!==key||"forceDirectedGraph"!==options.chart.type)&&(void 0===options.chart[key]||null===options.chart[key]?scope._config.extended&&("barColor"===key?options.chart[key]=value()():options.chart[key]=value()):scope.chart[key](options.chart[key]))))}),scope.api.updateWithData(),(options.title||scope._config.extended)&&configureWrapper("title"),(options.subtitle||scope._config.extended)&&configureWrapper("subtitle"),(options.caption||scope._config.extended)&&configureWrapper("caption"),(options.styles||scope._config.extended)&&configureStyles(),nv.addGraph(function(){return scope.chart?(scope.chart.resizeHandler&&scope.chart.resizeHandler.clear(),scope.chart.resizeHandler=nv.utils.windowResize(function(){scope.chart&&scope.chart.update&&scope.chart.update()}),void 0!==options.chart.zoom&&["scatterChart","lineChart","candlestickBarChart","cumulativeLineChart","his
toricalBarChart","ohlcBarChart","stackedAreaChart"].indexOf(options.chart.type)>-1&&nvd3Utils.zoom(scope,options),scope.chart):void 0},options.chart.callback))},updateWithData:function(data){if(arguments.length){if(scope.data=data,scope._config.deepWatchData&&!scope._config.disabled)return}else data="sunburstChart"===scope.options.chart.type?angular.copy(scope.data):scope.data;if(data){d3.select(element[0]).select("svg").remove();var h,w;scope.svg=d3.select(element[0]).insert("svg",".caption"),(h=scope.options.chart.height)&&(isNaN(+h)||(h+="px"),scope.svg.attr("height",h).style({height:h})),(w=scope.options.chart.width)?(isNaN(+w)||(w+="px"),scope.svg.attr("width",w).style({width:w})):scope.svg.attr("width","100%").style({width:"100%"}),scope.svg.datum(data).call(scope.chart),scope.chart&&scope.chart.zoomRender&&scope.chart.zoomRender()}},clearElement:function(){if(element.find(".title").remove(),element.find(".subtitle").remove(),element.find(".caption").remove(),element.empty(),s
cope.chart&&scope.chart.tooltip&&scope.chart.tooltip.id&&d3.select("#"+scope.chart.tooltip.id()).remove(),nv.graphs&&scope.chart)for(var i=nv.graphs.length-1;i>=0;i--)nv.graphs[i]&&nv.graphs[i].id===scope.chart.id&&nv.graphs.splice(i,1);nv.tooltip&&nv.tooltip.cleanup&&nv.tooltip.cleanup(),scope.chart&&scope.chart.resizeHandler&&scope.chart.resizeHandler.clear(),scope.chart=null},getScope:function(){return scope},getElement:function(){return element}},scope._config.deepWatchOptions&&scope.$watch("options",nvd3Utils.debounce(function(newOptions){scope._config.disabled||scope.api.refresh()},scope._config.debounce,scope._config.debounceImmediate),!0),scope._config.deepWatchData&&(1===scope._config.deepWatchDataDepth?scope.$watchCollection("data",dataWatchFn):scope.$watch("data",dataWatchFn,2===scope._config.deepWatchDataDepth)),scope.$watch("config",function(newConfig,oldConfig){newConfig!==oldConfig&&(scope._config=angular.extend(defaultConfig,newConfig),scope.api.refresh())},!0),scope
._config.deepWatchOptions||scope._config.deepWatchData||scope.api.refresh(),angular.forEach(scope.events,function(eventHandler,event){scope.$on(event,function(e,args){return eventHandler(e,scope,args)})}),element.on("$destroy",function(){scope.api.clearElement()}),scope.$watch("isReady",function(isReady){isReady&&scope.onReady&&"function"==typeof scope.onReady()&&scope.onReady()(scope,element)})}}}]).factory("nvd3Utils",function(){return{debounce:function(func,wait,immediate){var timeout;return function(){var context=this,args=arguments,later=function(){timeout=null,immediate||func.apply(context,args)},callNow=immediate&&!timeout;clearTimeout(timeout),timeout=setTimeout(later,wait),callNow&&func.apply(context,args)}},deepExtend:function(dst){var me=this;return angular.forEach(arguments,function(obj){obj!==dst&&angular.forEach(obj,function(value,key){dst[key]&&dst[key].constructor&&dst[key].constructor===Object?me.deepExtend(dst[key],value):dst[key]=value})}),dst},zoom:function(scope
,options){var zoom=options.chart.zoom,enabled="undefined"==typeof zoom.enabled||null===zoom.enabled?!0:zoom.enabled;if(enabled){var fixDomain,d3zoom,zoomed,unzoomed,zoomend,xScale=scope.chart.xAxis.scale(),yScale=scope.chart.yAxis.scale(),xDomain=scope.chart.xDomain||xScale.domain,yDomain=scope.chart.yDomain||yScale.domain,x_boundary=xScale.domain().slice(),y_boundary=yScale.domain().slice(),scale=zoom.scale||1,translate=zoom.translate||[0,0],scaleExtent=zoom.scaleExtent||[1,10],useFixedDomain=zoom.useFixedDomain||!1,useNiceScale=zoom.useNiceScale||!1,horizontalOff=zoom.horizontalOff||!1,verticalOff=zoom.verticalOff||!1,unzoomEventType=zoom.unzoomEventType||"dblclick.zoom";useNiceScale&&(xScale.nice(),yScale.nice()),fixDomain=function(domain,boundary){return domain[0]=Math.min(Math.max(domain[0],boundary[0]),boundary[1]-boundary[1]/scaleExtent[1]),domain[1]=Math.max(boundary[0]+boundary[1]/scaleExtent[1],Math.min(domain[1],boundary[1])),domain},zoomed=function(){if(void 0!==zoom.zoo
med){var domains=zoom.zoomed(xScale.domain(),yScale.domain());horizontalOff||xDomain([domains.x1,domains.x2]),verticalOff||yDomain([domains.y1,domains.y2])}else horizontalOff||xDomain(useFixedDomain?fixDomain(xScale.domain(),x_boundary):xScale.domain()),verticalOff||yDomain(useFixedDomain?fixDomain(yScale.domain(),y_boundary):yScale.domain());scope.chart&&scope.chart.update()},unzoomed=function(){if(void 0!==zoom.unzoomed){var domains=zoom.unzoomed(xScale.domain(),yScale.domain());horizontalOff||xDomain([domains.x1,domains.x2]),verticalOff||yDomain([domains.y1,domains.y2])}else horizontalOff||xDomain(x_boundary),verticalOff||yDomain(y_boundary);d3zoom.scale(scale).translate(translate),scope.chart&&scope.chart.update()},zoomend=function(){void 0!==zoom.zoomend&&zoom.zoomend()},d3zoom=d3.behavior.zoom().x(xScale).y(yScale).scaleExtent(scaleExtent).on("zoom",zoomed).on("zoomend",zoomend),scope.svg&&(scope.svg.call(d3zoom),d3zoom.scale(scale).translate(translate).event(scope.svg),"none"
!==unzoomEventType&&scope.svg.on(unzoomEventType,unzoomed)),scope.chart&&(scope.chart.zoomRender=function(){d3zoom.scale(scale).translate(translate),xScale=scope.chart.xAxis.scale(),yScale=scope.chart.yAxis.scale(),xDomain=scope.chart.xDomain||xScale.domain,yDomain=scope.chart.yDomain||yScale.domain,x_boundary=xScale.domain().slice(),y_boundary=yScale.domain().slice(),d3zoom.x(xScale).y(yScale),scope.svg.call(d3zoom),"none"!==unzoomEventType&&scope.svg.on(unzoomEventType,unzoomed)})}}}})}(window);
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
deleted file mode 100644
index 3f985d1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-route-1.6.4.min.js
+++ /dev/null
@@ -1,17 +0,0 @@
-/*
- AngularJS v1.6.4
- (c) 2010-2017 Google, Inc. http://angularjs.org
- License: MIT
-*/
-(function(J,d){'use strict';function A(d){k&&d.get("$route")}function B(t,u,g){return{restrict:"ECA",terminal:!0,priority:400,transclude:"element",link:function(a,f,b,c,m){function v(){l&&(g.cancel(l),l=null);n&&(n.$destroy(),n=null);p&&(l=g.leave(p),l.done(function(a){!1!==a&&(l=null)}),p=null)}function E(){var b=t.current&&t.current.locals;if(d.isDefined(b&&b.$template)){var b=a.$new(),c=t.current;p=m(b,function(b){g.enter(b,null,p||f).done(function(b){!1===b||!d.isDefined(w)||w&&!a.$eval(w)||u()});
-v()});n=c.scope=b;n.$emit("$viewContentLoaded");n.$eval(k)}else v()}var n,p,l,w=b.autoscroll,k=b.onload||"";a.$on("$routeChangeSuccess",E);E()}}}function C(d,k,g){return{restrict:"ECA",priority:-400,link:function(a,f){var b=g.current,c=b.locals;f.html(c.$template);var m=d(f.contents());if(b.controller){c.$scope=a;var v=k(b.controller,c);b.controllerAs&&(a[b.controllerAs]=v);f.data("$ngControllerController",v);f.children().data("$ngControllerController",v)}a[b.resolveAs||"$resolve"]=c;m(a)}}}var x,
-y,F,G,z=d.module("ngRoute",[]).info({angularVersion:"1.6.4"}).provider("$route",function(){function t(a,f){return d.extend(Object.create(a),f)}function u(a,d){var b=d.caseInsensitiveMatch,c={originalPath:a,regexp:a},g=c.keys=[];a=a.replace(/([().])/g,"\\$1").replace(/(\/)?:(\w+)(\*\?|[?*])?/g,function(a,b,d,c){a="?"===c||"*?"===c?"?":null;c="*"===c||"*?"===c?"*":null;g.push({name:d,optional:!!a});b=b||"";return""+(a?"":b)+"(?:"+(a?b:"")+(c&&"(.+?)"||"([^/]+)")+(a||"")+")"+(a||"")}).replace(/([/$*])/g,
-"\\$1");c.regexp=new RegExp("^"+a+"$",b?"i":"");return c}x=d.isArray;y=d.isObject;F=d.isDefined;G=d.noop;var g={};this.when=function(a,f){var b;b=void 0;if(x(f)){b=b||[];for(var c=0,m=f.length;c<m;c++)b[c]=f[c]}else if(y(f))for(c in b=b||{},f)if("$"!==c.charAt(0)||"$"!==c.charAt(1))b[c]=f[c];b=b||f;d.isUndefined(b.reloadOnSearch)&&(b.reloadOnSearch=!0);d.isUndefined(b.caseInsensitiveMatch)&&(b.caseInsensitiveMatch=this.caseInsensitiveMatch);g[a]=d.extend(b,a&&u(a,b));a&&(c="/"===a[a.length-1]?a.substr(0,
-a.length-1):a+"/",g[c]=d.extend({redirectTo:a},u(c,b)));return this};this.caseInsensitiveMatch=!1;this.otherwise=function(a){"string"===typeof a&&(a={redirectTo:a});this.when(null,a);return this};k=!0;this.eagerInstantiationEnabled=function(a){return F(a)?(k=a,this):k};this.$get=["$rootScope","$location","$routeParams","$q","$injector","$templateRequest","$sce","$browser",function(a,f,b,c,m,k,u,n){function p(e){var h=q.current;(y=(s=C())&&h&&s.$$route===h.$$route&&d.equals(s.pathParams,h.pathParams)&&
-!s.reloadOnSearch&&!D)||!h&&!s||a.$broadcast("$routeChangeStart",s,h).defaultPrevented&&e&&e.preventDefault()}function l(){var e=q.current,h=s;if(y)e.params=h.params,d.copy(e.params,b),a.$broadcast("$routeUpdate",e);else if(h||e){D=!1;q.current=h;var H=c.resolve(h);n.$$incOutstandingRequestCount();H.then(w).then(z).then(function(c){return c&&H.then(A).then(function(c){h===q.current&&(h&&(h.locals=c,d.copy(h.params,b)),a.$broadcast("$routeChangeSuccess",h,e))})}).catch(function(b){h===q.current&&a.$broadcast("$routeChangeError",
-h,e,b)}).finally(function(){n.$$completeOutstandingRequest(G)})}}function w(e){var a={route:e,hasRedirection:!1};if(e)if(e.redirectTo)if(d.isString(e.redirectTo))a.path=x(e.redirectTo,e.params),a.search=e.params,a.hasRedirection=!0;else{var b=f.path(),g=f.search();e=e.redirectTo(e.pathParams,b,g);d.isDefined(e)&&(a.url=e,a.hasRedirection=!0)}else if(e.resolveRedirectTo)return c.resolve(m.invoke(e.resolveRedirectTo)).then(function(e){d.isDefined(e)&&(a.url=e,a.hasRedirection=!0);return a});return a}
-function z(a){var b=!0;if(a.route!==q.current)b=!1;else if(a.hasRedirection){var d=f.url(),c=a.url;c?f.url(c).replace():c=f.path(a.path).search(a.search).replace().url();c!==d&&(b=!1)}return b}function A(a){if(a){var b=d.extend({},a.resolve);d.forEach(b,function(a,e){b[e]=d.isString(a)?m.get(a):m.invoke(a,null,null,e)});a=B(a);d.isDefined(a)&&(b.$template=a);return c.all(b)}}function B(a){var b,c;d.isDefined(b=a.template)?d.isFunction(b)&&(b=b(a.params)):d.isDefined(c=a.templateUrl)&&(d.isFunction(c)&&
-(c=c(a.params)),d.isDefined(c)&&(a.loadedTemplateUrl=u.valueOf(c),b=k(c)));return b}function C(){var a,b;d.forEach(g,function(c,g){var r;if(r=!b){var k=f.path();r=c.keys;var m={};if(c.regexp)if(k=c.regexp.exec(k)){for(var l=1,n=k.length;l<n;++l){var p=r[l-1],q=k[l];p&&q&&(m[p.name]=q)}r=m}else r=null;else r=null;r=a=r}r&&(b=t(c,{params:d.extend({},f.search(),a),pathParams:a}),b.$$route=c)});return b||g[null]&&t(g[null],{params:{},pathParams:{}})}function x(a,b){var c=[];d.forEach((a||"").split(":"),
-function(a,d){if(0===d)c.push(a);else{var e=a.match(/(\w+)(?:[?*])?(.*)/),f=e[1];c.push(b[f]);c.push(e[2]||"");delete b[f]}});return c.join("")}var D=!1,s,y,q={routes:g,reload:function(){D=!0;var b={defaultPrevented:!1,preventDefault:function(){this.defaultPrevented=!0;D=!1}};a.$evalAsync(function(){p(b);b.defaultPrevented||l()})},updateParams:function(a){if(this.current&&this.current.$$route)a=d.extend({},this.current.params,a),f.path(x(this.current.$$route.originalPath,a)),f.search(a);else throw I("norout");
-}};a.$on("$locationChangeStart",p);a.$on("$locationChangeSuccess",l);return q}]}).run(A),I=d.$$minErr("ngRoute"),k;A.$inject=["$injector"];z.provider("$routeParams",function(){this.$get=function(){return{}}});z.directive("ngView",B);z.directive("ngView",C);B.$inject=["$route","$anchorScroll","$animate"];C.$inject=["$compile","$controller","$route"]})(window,window.angular);
-//# sourceMappingURL=angular-route.min.js.map
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
deleted file mode 100644
index 84741f4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneBucket.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.FileEntity;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClientBuilder;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Strings;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.util.LinkedList;
-import java.util.List;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.ENCODING;
-import static org.apache.hadoop.ozone.web.utils.OzoneUtils.ENCODING_NAME;
-
-/**
- * A Bucket class the represents an Ozone Bucket.
- */
-public class OzoneBucket {
- static final Logger LOG = LoggerFactory.getLogger(OzoneBucket.class);
-
- private BucketInfo bucketInfo;
- private OzoneVolume volume;
-
- /**
- * Constructor for bucket.
- *
- * @param info - BucketInfo
- * @param volume - OzoneVolume Object that contains this bucket
- */
- public OzoneBucket(BucketInfo info, OzoneVolume volume) {
- this.bucketInfo = info;
- this.volume = volume;
- }
-
- /**
- * Gets bucket Info.
- *
- * @return BucketInfo
- */
- public BucketInfo getBucketInfo() {
- return bucketInfo;
- }
-
- /**
- * Sets Bucket Info.
- *
- * @param bucketInfo BucketInfo
- */
- public void setBucketInfo(BucketInfo bucketInfo) {
- this.bucketInfo = bucketInfo;
- }
-
- /**
- * Returns the parent volume class.
- *
- * @return - OzoneVolume
- */
- OzoneVolume getVolume() {
- return volume;
- }
-
- /**
- * Returns bucket name.
- *
- * @return Bucket Name
- */
- public String getBucketName() {
- return bucketInfo.getBucketName();
- }
-
- /**
- * Returns the Acls on the bucket.
- *
- * @return - Acls
- */
- public List<OzoneAcl> getAcls() {
- return bucketInfo.getAcls();
- }
-
- /**
- * Return versioning info on the bucket - Enabled or disabled.
- *
- * @return - Version Enum
- */
- public OzoneConsts.Versioning getVersioning() {
- return bucketInfo.getVersioning();
- }
-
- /**
- * Gets the Storage class for the bucket.
- *
- * @return Storage Class Enum
- */
- public StorageType getStorageType() {
- return bucketInfo.getStorageType();
- }
-
- /**
- * Gets the creation time of the bucket.
- *
- * @return String
- */
- public String getCreatedOn() {
- return bucketInfo.getCreatedOn();
- }
-
- /**
- * Puts an Object in Ozone bucket.
- *
- * @param keyName - Name of the key
- * @param data - Data that you want to put
- * @throws OzoneException
- */
- public void putKey(String keyName, String data) throws OzoneException {
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name.");
- }
-
- if (data == null) {
- throw new OzoneRestClientException("Invalid data.");
- }
-
- HttpPut putRequest = null;
- InputStream is = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName).build();
-
- putRequest = getVolume().getClient().getHttpPut(builder.toString());
-
- is = new ByteArrayInputStream(data.getBytes(ENCODING));
- putRequest.setEntity(new InputStreamEntity(is, data.length()));
- is.mark(data.length());
- try {
- putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(is));
- } finally {
- is.reset();
- }
- executePutKey(putRequest, httpClient);
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- IOUtils.closeStream(is);
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Puts an Object in Ozone Bucket.
- *
- * @param dataFile - File from which you want the data to be put. Key Name
- * will same as the file name, devoid of any path.
- * @throws OzoneException
- */
- public void putKey(File dataFile) throws OzoneException {
- if (dataFile == null) {
- throw new OzoneRestClientException("Invalid file object.");
- }
- String keyName = dataFile.getName();
- putKey(keyName, dataFile);
- }
-
- /**
- * Puts a Key in Ozone Bucket.
- *
- * @param keyName - Name of the Key
- * @param file - Stream that gets read to be put into Ozone.
- * @throws OzoneException
- */
- public void putKey(String keyName, File file)
- throws OzoneException {
-
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- if (file == null) {
- throw new OzoneRestClientException("Invalid data stream");
- }
-
- HttpPut putRequest = null;
- FileInputStream fis = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName).build();
-
- putRequest = getVolume().getClient().getHttpPut(builder.toString());
-
- FileEntity fileEntity = new FileEntity(file, ContentType
- .APPLICATION_OCTET_STREAM);
- putRequest.setEntity(fileEntity);
-
- fis = new FileInputStream(file);
- putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
- executePutKey(putRequest, httpClient);
-
- } catch (IOException | URISyntaxException ex) {
- final OzoneRestClientException orce = new OzoneRestClientException(
- "Failed to putKey: keyName=" + keyName + ", file=" + file);
- orce.initCause(ex);
- LOG.trace("", orce);
- throw orce;
- } finally {
- IOUtils.closeStream(fis);
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * executePutKey executes the Put request against the Ozone Server.
- *
- * @param putRequest - Http Put Request
- * @param httpClient - httpClient
- * @throws OzoneException
- * @throws IOException
- */
- public static void executePutKey(HttpPut putRequest,
- CloseableHttpClient httpClient) throws OzoneException, IOException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(putRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return;
- }
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- throw OzoneException.parse(EntityUtils.toString(entity));
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Gets a key from the Ozone server and writes to the file pointed by the
- * downloadTo PAth.
- *
- * @param keyName - Key Name in Ozone.
- * @param downloadTo File Name to download the Key's Data to
- */
- public void getKey(String keyName, Path downloadTo) throws OzoneException {
-
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- if (downloadTo == null) {
- throw new OzoneRestClientException("Invalid download path");
- }
-
- FileOutputStream outPutFile = null;
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- outPutFile = new FileOutputStream(downloadTo.toFile());
-
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName).build();
-
- getRequest = getVolume().getClient().getHttpGet(builder.toString());
- executeGetKey(getRequest, httpClient, outPutFile);
- outPutFile.flush();
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- IOUtils.closeStream(outPutFile);
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * Returns the data part of the key as a string.
- *
- * @param keyName - KeyName to get
- * @return String - Data
- * @throws OzoneException
- */
- public String getKey(String keyName) throws OzoneException {
-
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- HttpGet getRequest = null;
- ByteArrayOutputStream outPutStream = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- outPutStream = new ByteArrayOutputStream();
-
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
-
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName).build();
-
- getRequest = getVolume().getClient().getHttpGet(builder.toString());
- executeGetKey(getRequest, httpClient, outPutStream);
- return outPutStream.toString(ENCODING_NAME);
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- IOUtils.closeStream(outPutStream);
- OzoneClientUtils.releaseConnection(getRequest);
- }
-
- }
-
- /**
- * Executes get key and returns the data.
- *
- * @param getRequest - http Get Request
- * @param httpClient - Client
- * @param stream - Stream to write data to.
- * @throws IOException
- * @throws OzoneException
- */
- public static void executeGetKey(HttpGet getRequest,
- CloseableHttpClient httpClient, OutputStream stream)
- throws IOException, OzoneException {
-
- HttpEntity entity = null;
- try {
-
- HttpResponse response = httpClient.execute(getRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (errorCode == HTTP_OK) {
- entity.writeTo(stream);
- return;
- }
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- throw OzoneException.parse(EntityUtils.toString(entity));
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Deletes a key in this bucket.
- *
- * @param keyName - Name of the Key
- * @throws OzoneException
- */
- public void deleteKey(String keyName) throws OzoneException {
-
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- HttpDelete deleteRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName).build();
-
- deleteRequest = getVolume()
- .getClient().getHttpDelete(builder.toString());
- executeDeleteKey(deleteRequest, httpClient);
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(deleteRequest);
- }
- }
-
- /**
- * Executes deleteKey.
- *
- * @param deleteRequest - http Delete Request
- * @param httpClient - Client
- * @throws IOException
- * @throws OzoneException
- */
- private void executeDeleteKey(HttpDelete deleteRequest,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
-
- HttpEntity entity = null;
- try {
-
- HttpResponse response = httpClient.execute(deleteRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (errorCode == HTTP_OK) {
- return;
- }
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- throw OzoneException.parse(EntityUtils.toString(entity));
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * List all keys in a bucket.
- *
- * @param resultLength The max length of listing result.
- * @param previousKey The key from where listing should start,
- * this key is excluded in the result.
- * @param prefix The prefix that return list keys start with.
- * @return List of OzoneKeys
- * @throws OzoneException
- */
- public List<OzoneKey> listKeys(String resultLength, String previousKey,
- String prefix) throws OzoneException {
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- OzoneRestClient client = getVolume().getClient();
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder.setPath("/" + getVolume().getVolumeName() + "/" + getBucketName())
- .build();
-
- if (!Strings.isNullOrEmpty(resultLength)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
- }
-
- if (!Strings.isNullOrEmpty(previousKey)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousKey);
- }
-
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
- }
-
- final String uri = builder.toString();
- getRequest = client.getHttpGet(uri);
- LOG.trace("listKeys URI={}", uri);
- return executeListKeys(getRequest, httpClient);
-
- } catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * List keys in a bucket with the provided prefix, with paging results.
- *
- * @param prefix The prefix of the object keys
- * @param maxResult max size per response
- * @param prevKey the previous key for paging
- */
- public List<OzoneKey> listKeys(String prefix, int maxResult, String prevKey)
- throws OzoneException {
- HttpGet getRequest = null;
- try {
- final URI uri = new URIBuilder(volume.getClient().getEndPointURI())
- .setPath(OzoneConsts.KSM_KEY_PREFIX + getVolume().getVolumeName() +
- OzoneConsts.KSM_KEY_PREFIX + getBucketName())
- .setParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix)
- .setParameter(Header.OZONE_LIST_QUERY_MAXKEYS,
- String.valueOf(maxResult))
- .setParameter(Header.OZONE_LIST_QUERY_PREVKEY, prevKey)
- .build();
- final OzoneRestClient client = getVolume().getClient();
- getRequest = client.getHttpGet(uri.toString());
- return executeListKeys(getRequest, HttpClientBuilder.create().build());
- } catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage());
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * Execute list Key.
- *
- * @param getRequest - HttpGet
- * @param httpClient - HttpClient
- * @return List<OzoneKey>
- * @throws IOException
- * @throws OzoneException
- */
- public static List<OzoneKey> executeListKeys(HttpGet getRequest,
- CloseableHttpClient httpClient) throws IOException, OzoneException {
- HttpEntity entity = null;
- List<OzoneKey> ozoneKeyList = new LinkedList<OzoneKey>();
- try {
- HttpResponse response = httpClient.execute(getRequest);
- int errorCode = response.getStatusLine().getStatusCode();
-
- entity = response.getEntity();
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
- if (errorCode == HTTP_OK) {
- String temp = EntityUtils.toString(entity);
- ListKeys keyList = ListKeys.parse(temp);
-
- for (KeyInfo info : keyList.getKeyList()) {
- ozoneKeyList.add(new OzoneKey(info));
- }
- return ozoneKeyList;
-
- } else {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Get info of the specified key.
- */
- public OzoneKey getKeyInfo(String keyName) throws OzoneException {
- if ((keyName == null) || keyName.isEmpty()) {
- throw new OzoneRestClientException(
- "Unable to get key info, key name is null or empty");
- }
-
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- OzoneRestClient client = getVolume().getClient();
- URIBuilder builder = new URIBuilder(volume.getClient().getEndPointURI());
- builder
- .setPath("/" + getVolume().getVolumeName() + "/" + getBucketName()
- + "/" + keyName)
- .setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_KEY)
- .build();
-
- getRequest = client.getHttpGet(builder.toString());
- return executeGetKeyInfo(getRequest, httpClient);
- } catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * Execute get Key info.
- *
- * @param getRequest - HttpGet
- * @param httpClient - HttpClient
- * @return List<OzoneKey>
- * @throws IOException
- * @throws OzoneException
- */
- private OzoneKey executeGetKeyInfo(HttpGet getRequest,
- CloseableHttpClient httpClient) throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(getRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- if (errorCode == HTTP_OK) {
- OzoneKey key = new OzoneKey(
- KeyInfo.parse(EntityUtils.toString(entity)));
- return key;
- }
- throw OzoneException.parse(EntityUtils.toString(entity));
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
deleted file mode 100644
index 5a3a0c4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneKey.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-
-/**
- * Client side representation of an ozone Key.
- */
-public class OzoneKey {
- private KeyInfo keyInfo;
-
- /**
- * Constructor for Ozone Key.
- * @param keyInfo - Key Info
- */
- public OzoneKey(KeyInfo keyInfo) {
- this.keyInfo = keyInfo;
- }
-
- /**
- * Returns Key Info.
- * @return Object Info
- */
- public KeyInfo getObjectInfo() {
- return keyInfo;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
deleted file mode 100644
index 16a2e58..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClient.java
+++ /dev/null
@@ -1,801 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.ContentType;
-import org.apache.http.entity.FileEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.Closeable;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Locale;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone client that connects to an Ozone server. Please note that this class is
- * not thread safe.
- */
-public class OzoneRestClient implements Closeable {
- private URI endPointURI;
- private String userAuth;
-
- /**
- * Constructor for OzoneRestClient.
- */
- public OzoneRestClient() {
- }
-
- /**
- * Constructor for OzoneRestClient.
- */
- public OzoneRestClient(String ozoneURI)
- throws OzoneException, URISyntaxException {
- setEndPoint(ozoneURI);
- }
-
- /**
- * Constructor for OzoneRestClient.
- */
- public OzoneRestClient(String ozoneURI, String userAuth)
- throws OzoneException, URISyntaxException {
- setEndPoint(ozoneURI);
- setUserAuth(userAuth);
- }
-
- /**
- * Returns the end Point.
- *
- * @return String
- */
- public URI getEndPointURI() {
- return endPointURI;
- }
-
- /**
- * Sets the End Point info using an URI.
- *
- * @param endPointURI - URI
- * @throws OzoneException
- */
- public void setEndPointURI(URI endPointURI) throws OzoneException {
- if ((endPointURI == null) || (endPointURI.toString().isEmpty())) {
- throw new OzoneRestClientException("Invalid ozone URI");
- }
- this.endPointURI = endPointURI;
- }
-
- /**
- * Set endPoint.
- *
- * @param clusterFQDN - cluster FQDN.
- */
- public void setEndPoint(String clusterFQDN) throws
- OzoneException, URISyntaxException {
- setEndPointURI(new URI(clusterFQDN));
- }
-
- /**
- * Get user Auth String.
- *
- * @return - User Auth String
- */
- public String getUserAuth() {
- return this.userAuth;
- }
-
- /**
- * Set User Auth.
- *
- * @param userAuth - User Auth String
- */
- public void setUserAuth(String userAuth) {
- this.userAuth = userAuth;
- }
-
- /**
- * create volume.
- *
- * @param volumeName - volume name 3 - 63 chars, small letters.
- * @param onBehalfOf - The user on behalf we are making the call for
- * @param quota - Quota's are specified in a specific format. it is
- * integer(MB|GB|TB), for example 100TB.
- * @throws OzoneRestClientException
- */
- public OzoneVolume createVolume(String volumeName, String onBehalfOf,
- String quota) throws OzoneException {
- HttpPost httpPost = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(volumeName);
-
- URIBuilder builder = new URIBuilder(endPointURI);
- builder.setPath("/" + volumeName);
- if (quota != null) {
- builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota);
- }
-
- httpPost = getHttpPost(onBehalfOf, builder.build().toString());
- executeCreateVolume(httpPost, httpClient);
- return getVolume(volumeName);
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpPost);
- }
- }
-
- /**
- * Returns information about an existing Volume. if the Volume does not exist,
- * or if the user does not have access rights OzoneException is thrown
- *
- * @param volumeName - volume name 3 - 63 chars, small letters.
- * @return OzoneVolume Ozone Client Volume Class.
- * @throws OzoneException
- */
- public OzoneVolume getVolume(String volumeName) throws OzoneException {
- HttpGet httpGet = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(endPointURI);
- builder.setPath("/" + volumeName)
- .setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_VOLUME)
- .build();
-
- httpGet = getHttpGet(builder.toString());
- return executeInfoVolume(httpGet, httpClient);
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpGet);
- }
- }
-
- /**
- * List all the volumes owned by the user or Owned by the user specified in
- * the behalf of string.
- *
- * @param onBehalfOf
- * User Name of the user if it is not the caller. for example,
- * an admin wants to list some other users volumes.
- * @param prefix
- * Return only volumes that match this prefix.
- * @param maxKeys
- * Maximum number of results to return, if the result set
- * is smaller than requested size, it means that list is
- * complete.
- * @param previousVolume
- * The previous volume name.
- * @return List of Volumes
- * @throws OzoneException
- */
- public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
- int maxKeys, String previousVolume) throws OzoneException {
- HttpGet httpGet = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- URIBuilder builder = new URIBuilder(endPointURI);
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
- }
-
- if (maxKeys > 0) {
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, Integer
- .toString(maxKeys));
- }
-
- if (!Strings.isNullOrEmpty(previousVolume)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
- previousVolume);
- }
-
- builder.setPath("/").build();
-
- httpGet = getHttpGet(builder.toString());
- if (onBehalfOf != null) {
- httpGet.addHeader(Header.OZONE_USER, onBehalfOf);
- }
- return executeListVolume(httpGet, httpClient);
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpGet);
- }
- }
-
- /**
- * List all the volumes owned by the user or Owned by the user specified in
- * the behalf of string.
- *
- * @param onBehalfOf - User Name of the user if it is not the caller. for
- * example, an admin wants to list some other users
- * volumes.
- * @param prefix - Return only volumes that match this prefix.
- * @param maxKeys - Maximum number of results to return, if the result set
- * is smaller than requested size, it means that list is
- * complete.
- * @param prevKey - The last key that client got, server will continue
- * returning results from that point.
- * @return List of Volumes
- * @throws OzoneException
- */
- public List<OzoneVolume> listVolumes(String onBehalfOf, String prefix,
- int maxKeys, OzoneVolume prevKey) throws OzoneException {
- String volumeName = null;
-
- if (prevKey != null) {
- volumeName = prevKey.getVolumeName();
- }
-
- return listVolumes(onBehalfOf, prefix, maxKeys, volumeName);
- }
-
- /**
- * List volumes of the current user or if onBehalfof is not null lists volume
- * owned by that user. You need admin privilege to read other users volume
- * lists.
- *
- * @param onBehalfOf - Name of the user you want to get volume list
- * @return - Volume list.
- * @throws OzoneException
- */
- public List<OzoneVolume> listVolumes(String onBehalfOf)
- throws OzoneException {
- return listVolumes(onBehalfOf, null,
- Integer.parseInt(Header.OZONE_DEFAULT_LIST_SIZE), StringUtils.EMPTY);
- }
-
- /**
- * List all volumes in a cluster. This can be invoked only by an Admin.
- *
- * @param prefix - Returns only volumes that match this prefix.
- * @param maxKeys - Maximum niumber of keys to return
- * @param prevKey - Last Ozone Volume from the last Iteration.
- * @return List of Volumes
- * @throws OzoneException
- */
- public List<OzoneVolume> listAllVolumes(String prefix, int maxKeys,
- OzoneVolume prevKey) throws OzoneException {
- HttpGet httpGet = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- URIBuilder builder = new URIBuilder(endPointURI);
- if (prefix != null) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
- }
-
- if (maxKeys > 0) {
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, Integer
- .toString(maxKeys));
- }
-
- if (prevKey != null) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY,
- prevKey.getOwnerName()+ "/" + prevKey.getVolumeName());
- }
-
- builder.addParameter(Header.OZONE_LIST_QUERY_ROOTSCAN, "true");
- builder.setPath("/").build();
- httpGet = getHttpGet(builder.toString());
- return executeListVolume(httpGet, httpClient);
-
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpGet);
- }
- }
-
- /**
- * delete a given volume.
- *
- * @param volumeName - volume to be deleted.
- * @throws OzoneException - Ozone Exception
- */
- public void deleteVolume(String volumeName) throws OzoneException {
- HttpDelete httpDelete = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(endPointURI);
- builder.setPath("/" + volumeName).build();
-
- httpDelete = getHttpDelete(builder.toString());
- executeDeleteVolume(httpDelete, httpClient);
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpDelete);
- }
- }
-
- /**
- * Sets the Volume Owner.
- *
- * @param volumeName - Volume Name
- * @param newOwner - New Owner Name
- * @throws OzoneException
- */
- public void setVolumeOwner(String volumeName, String newOwner)
- throws OzoneException {
- HttpPut putRequest = null;
- if (newOwner == null || newOwner.isEmpty()) {
- throw new OzoneRestClientException("Invalid new owner name");
- }
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(endPointURI);
- builder.setPath("/" + volumeName).build();
-
- putRequest = getHttpPut(builder.toString());
- putRequest.addHeader(Header.OZONE_USER, newOwner);
- executePutVolume(putRequest, httpClient);
-
- } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Sets the Volume Quota. Quota's are specified in a specific format. it is
- * <integer>|(MB|GB|TB. for example 100TB.
- * <p>
- * To Remove a quota you can specify Header.OZONE_QUOTA_REMOVE
- *
- * @param volumeName - volume name
- * @param quota - Quota String or Header.OZONE_QUOTA_REMOVE
- * @throws OzoneException
- */
- public void setVolumeQuota(String volumeName, String quota)
- throws OzoneException {
- if (quota == null || quota.isEmpty()) {
- throw new OzoneRestClientException("Invalid quota");
- }
- HttpPut putRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(volumeName);
- URIBuilder builder = new URIBuilder(endPointURI);
- builder.setPath("/" + volumeName)
- .setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota)
- .build();
-
- putRequest = getHttpPut(builder.toString());
- executePutVolume(putRequest, httpClient);
-
- } catch (URISyntaxException | IllegalArgumentException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Sends the create Volume request to the server.
- *
- * @param httppost - http post class
- * @param httpClient - httpClient
- * @throws IOException -
- * @throws OzoneException
- */
- private void executeCreateVolume(HttpPost httppost,
- final CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(httppost);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return;
- }
-
- if (entity != null) {
- throw OzoneException.parse(EntityUtils.toString(entity));
- } else {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
- } finally {
- if (entity != null) {
- EntityUtils.consume(entity);
- }
- }
- }
-
- /**
- * Sends the create Volume request to the server.
- *
- * @param httpGet - httpGet
- * @return OzoneVolume
- * @throws IOException -
- * @throws OzoneException
- */
- private OzoneVolume executeInfoVolume(HttpGet httpGet,
- final CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(httpGet);
- int errorCode = response.getStatusLine().getStatusCode();
-
- entity = response.getEntity();
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- if (errorCode == HTTP_OK) {
- OzoneVolume volume = new OzoneVolume(this);
- volume.setVolumeInfo(EntityUtils.toString(entity));
- return volume;
- } else {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Sends update volume requests to the server.
- *
- * @param putRequest http request
- * @throws IOException
- * @throws OzoneException
- */
- private void executePutVolume(HttpPut putRequest,
- final CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(putRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
- if (errorCode != HTTP_OK) {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consume(entity);
- }
- }
- }
-
- /**
- * List Volumes.
- *
- * @param httpGet - httpGet
- * @return OzoneVolume
- * @throws IOException -
- * @throws OzoneException
- */
- private List<OzoneVolume> executeListVolume(HttpGet httpGet,
- final CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- List<OzoneVolume> volList = new LinkedList<>();
- try {
- HttpResponse response = httpClient.execute(httpGet);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
-
- String temp = EntityUtils.toString(entity);
- if (errorCode == HTTP_OK) {
- ListVolumes listVolumes =
- ListVolumes.parse(temp);
-
- for (VolumeInfo info : listVolumes.getVolumes()) {
- volList.add(new OzoneVolume(info, this));
- }
- return volList;
-
- } else {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Delete Volume.
- *
- * @param httpDelete - Http Delete Request
- * @throws IOException
- * @throws OzoneException
- */
- private void executeDeleteVolume(HttpDelete httpDelete,
- final CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(httpDelete);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (errorCode != HTTP_OK) {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Puts a Key in Ozone Bucket.
- *
- * @param volumeName - Name of the Volume
- * @param bucketName - Name of the Bucket
- * @param keyName - Name of the Key
- * @param file - Stream that gets read to be put into Ozone.
- * @throws OzoneException
- */
- public void putKey(String volumeName, String bucketName, String keyName,
- File file) throws OzoneException {
- OzoneUtils.verifyResourceName(volumeName);
- OzoneUtils.verifyResourceName(bucketName);
-
- if (StringUtils.isEmpty(keyName)) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- if (file == null) {
- throw new OzoneRestClientException("Invalid data stream");
- }
-
- HttpPut putRequest = null;
- FileInputStream fis = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- URIBuilder builder = new URIBuilder(getEndPointURI());
- builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName)
- .build();
-
- putRequest = getHttpPut(builder.toString());
-
- FileEntity fileEntity = new FileEntity(file, ContentType
- .APPLICATION_OCTET_STREAM);
- putRequest.setEntity(fileEntity);
-
- fis = new FileInputStream(file);
- putRequest.setHeader(Header.CONTENT_MD5, DigestUtils.md5Hex(fis));
- OzoneBucket.executePutKey(putRequest, httpClient);
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- IOUtils.closeStream(fis);
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Gets a key from the Ozone server and writes to the file pointed by the
- * downloadTo Path.
- *
- * @param volumeName - Volume Name in Ozone.
- * @param bucketName - Bucket Name in Ozone.
- * @param keyName - Key Name in Ozone.
- * @param downloadTo File Name to download the Key's Data to
- */
- public void getKey(String volumeName, String bucketName, String keyName,
- Path downloadTo) throws OzoneException {
- OzoneUtils.verifyResourceName(volumeName);
- OzoneUtils.verifyResourceName(bucketName);
-
- if (StringUtils.isEmpty(keyName)) {
- throw new OzoneRestClientException("Invalid key Name");
- }
-
- if (downloadTo == null) {
- throw new OzoneRestClientException("Invalid download path");
- }
-
- FileOutputStream outPutFile = null;
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- outPutFile = new FileOutputStream(downloadTo.toFile());
-
- URIBuilder builder = new URIBuilder(getEndPointURI());
- builder.setPath("/" + volumeName + "/" + bucketName + "/" + keyName)
- .build();
-
- getRequest = getHttpGet(builder.toString());
- OzoneBucket.executeGetKey(getRequest, httpClient, outPutFile);
- outPutFile.flush();
- } catch (IOException | URISyntaxException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- IOUtils.closeStream(outPutFile);
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * List all keys in the given bucket.
- *
- * @param volumeName - Volume name
- * @param bucketName - Bucket name
- * @param resultLength The max length of listing result.
- * @param previousKey The key from where listing should start,
- * this key is excluded in the result.
- * @param prefix The prefix that return list keys start with.
- *
- * @return List of OzoneKeys
- */
- public List<OzoneKey> listKeys(String volumeName, String bucketName,
- String resultLength, String previousKey, String prefix)
- throws OzoneException {
- OzoneUtils.verifyResourceName(volumeName);
- OzoneUtils.verifyResourceName(bucketName);
-
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = OzoneClientUtils.newHttpClient()) {
- URIBuilder builder = new URIBuilder(getEndPointURI());
- builder.setPath("/" + volumeName + "/" + bucketName).build();
-
- if (!Strings.isNullOrEmpty(resultLength)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
- }
-
- if (!Strings.isNullOrEmpty(previousKey)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousKey);
- }
-
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
- }
-
- getRequest = getHttpGet(builder.toString());
- return OzoneBucket.executeListKeys(getRequest, httpClient);
- } catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * Returns a standard HttpPost Object to use for ozone post requests.
- *
- * @param onBehalfOf - If the use is being made on behalf of user, that user
- * @param uriString - UriString
- * @return HttpPost
- */
- public HttpPost getHttpPost(String onBehalfOf, String uriString) {
- HttpPost httpPost = new HttpPost(uriString);
- addOzoneHeaders(httpPost);
- if (onBehalfOf != null) {
- httpPost.addHeader(Header.OZONE_USER, onBehalfOf);
- }
- return httpPost;
- }
-
- /**
- * Returns a standard HttpGet Object to use for ozone Get requests.
- *
- * @param uriString - The full Uri String
- * @return HttpGet
- */
- public HttpGet getHttpGet(String uriString) {
- HttpGet httpGet = new HttpGet(uriString);
- addOzoneHeaders(httpGet);
- return httpGet;
- }
-
- /**
- * Returns httpDelete.
- *
- * @param uriString - uri
- * @return HttpDelete
- */
- public HttpDelete getHttpDelete(String uriString) {
- HttpDelete httpDel = new HttpDelete(uriString);
- addOzoneHeaders(httpDel);
- return httpDel;
- }
-
- /**
- * returns an HttpPut Object.
- *
- * @param uriString - Uri
- * @return HttpPut
- */
- public HttpPut getHttpPut(String uriString) {
- HttpPut httpPut = new HttpPut(uriString);
- addOzoneHeaders(httpPut);
- return httpPut;
- }
-
- /**
- * Add Ozone Headers.
- *
- * @param httpRequest - Http Request
- */
- private void addOzoneHeaders(HttpRequestBase httpRequest) {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
-
- httpRequest.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- httpRequest.addHeader(HttpHeaders.DATE,
- format.format(new Date(Time.monotonicNow())));
- if (getUserAuth() != null) {
- httpRequest.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- getUserAuth());
- }
- }
-
- /**
- * Closes this stream and releases any system resources associated with it. If
- * the stream is already closed then invoking this method has no effect.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- // TODO : Currently we create a new HTTP client. We should switch
- // This to a Pool and cleanup the pool here.
- }
-
- @VisibleForTesting
- public CloseableHttpClient newHttpClient() {
- return OzoneClientUtils.newHttpClient();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
deleted file mode 100644
index dfb2357..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneRestClientException.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.client;
-
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-
-/**
- * This exception is thrown by the Ozone Clients.
- */
-public class OzoneRestClientException extends OzoneException {
- /**
- * Constructor that allows the shortMessage.
- *
- * @param shortMessage Short Message
- */
- public OzoneRestClientException(String shortMessage) {
- super(0, shortMessage, shortMessage);
- }
-
- /**
- * Constructor that allows a shortMessage and an exception.
- *
- * @param shortMessage short message
- * @param ex exception
- */
- public OzoneRestClientException(String shortMessage, Exception ex) {
- super(0, shortMessage, shortMessage, ex);
- }
-
- /**
- * Constructor that allows the shortMessage and a longer message.
- *
- * @param shortMessage Short Message
- * @param message long error message
- */
- public OzoneRestClientException(String shortMessage, String message) {
- super(0, shortMessage, message);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
deleted file mode 100644
index 4fac514..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/OzoneVolume.java
+++ /dev/null
@@ -1,581 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.client;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.util.EntityUtils;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone Volume Class.
- */
-public class OzoneVolume {
- private VolumeInfo volumeInfo;
- private Map<String, String> headerMap;
- private final OzoneRestClient client;
-
- /**
- * Constructor for OzoneVolume.
- */
- public OzoneVolume(OzoneRestClient client) {
- this.client = client;
- this.headerMap = new HashMap<>();
- }
-
- /**
- * Constructor for OzoneVolume.
- *
- * @param volInfo - volume Info.
- * @param client Client
- */
- public OzoneVolume(VolumeInfo volInfo, OzoneRestClient client) {
- this.volumeInfo = volInfo;
- this.client = client;
- }
-
- /**
- * Returns a Json String of this class.
- * @return String
- * @throws IOException
- */
- public String getJsonString() throws IOException {
- return volumeInfo.toJsonString();
- }
-
- /**
- * sets the Volume Info.
- *
- * @param volInfoString - Volume Info String
- */
- public void setVolumeInfo(String volInfoString) throws IOException {
- this.volumeInfo = VolumeInfo.parse(volInfoString);
- }
-
- /**
- * @return the volume info.
- */
- public VolumeInfo getVolumeInfo() {
- return this.volumeInfo;
- }
-
- /**
- * Returns volume Name.
- *
- * @return Volume Name.
- */
- public String getVolumeName() {
- return this.volumeInfo.getVolumeName();
- }
-
- /**
- * Get created by.
- *
- * @return String
- */
- public String getCreatedby() {
- return this.volumeInfo.getCreatedBy();
- }
-
- /**
- * returns the Owner name.
- *
- * @return String
- */
- public String getOwnerName() {
- return this.volumeInfo.getOwner().getName();
- }
-
- /**
- * Returns Quota Info.
- *
- * @return Quota
- */
- public OzoneQuota getQuota() {
- return volumeInfo.getQuota();
- }
-
- /**
- * Returns creation time of Volume.
- *
- * @return String
- */
- public String getCreatedOn() {
- return volumeInfo.getCreatedOn();
- }
-
- /**
- * Returns a Http header from the Last Volume related call.
- *
- * @param headerName - Name of the header
- * @return - Header Value
- */
- public String getHeader(String headerName) {
- return headerMap.get(headerName);
- }
-
- /**
- * Gets the Client, this is used by Bucket and Key Classes.
- *
- * @return - Ozone Client
- */
- OzoneRestClient getClient() {
- return client;
- }
-
- /**
- * Create Bucket - Creates a bucket under a given volume.
- *
- * @param bucketName - Bucket Name
- * @param acls - Acls - User Acls
- * @param storageType - Storage Class
- * @param versioning - enable versioning support on a bucket.
- *
- *
- * @return - a Ozone Bucket Object
- */
- public OzoneBucket createBucket(String bucketName, String[] acls,
- StorageType storageType,
- OzoneConsts.Versioning versioning)
- throws OzoneException {
-
- HttpPost httpPost = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(bucketName);
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-
- httpPost = client.getHttpPost(null, builder.toString());
- if (acls != null) {
- for (String acl : acls) {
- httpPost
- .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl);
- }
- }
-
- httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
- httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING, versioning.toString());
- executeCreateBucket(httpPost, httpClient);
- return getBucket(bucketName);
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(httpPost);
- }
- }
-
- /**
- * Create Bucket.
- *
- * @param bucketName - bucket name
- * @param acls - acls
- * @param storageType - storage class
- *
- * @throws OzoneException
- */
- public OzoneBucket createBucket(String bucketName, String[] acls,
- StorageType storageType)
- throws OzoneException {
- return createBucket(bucketName, acls, storageType,
- OzoneConsts.Versioning.DISABLED);
- }
-
- /**
- * Create Bucket.
- *
- * @param bucketName - bucket name
- * @param acls - acls
- *
- * @throws OzoneException
- */
- public OzoneBucket createBucket(String bucketName, String[] acls)
- throws OzoneException {
- return createBucket(bucketName, acls, StorageType.DEFAULT,
- OzoneConsts.Versioning.DISABLED);
- }
-
-
- /**
- * Create Bucket.
- *
- * @param bucketName - bucket name
- *
- * @throws OzoneException
- */
- public OzoneBucket createBucket(String bucketName) throws OzoneException {
- return createBucket(bucketName, null, StorageType.DEFAULT,
- OzoneConsts.Versioning.DISABLED);
- }
-
-
- /**
- * execute a Create Bucket Request against Ozone server.
- *
- * @param httppost - httpPost
- *
- * @throws IOException
- * @throws OzoneException
- */
- private void executeCreateBucket(HttpPost httppost,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(httppost);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return;
- }
-
- if (entity != null) {
- throw OzoneException.parse(EntityUtils.toString(entity));
- } else {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Adds Acls to an existing bucket.
- *
- * @param bucketName - Name of the bucket
- * @param acls - Acls
- *
- * @throws OzoneException
- */
- public void addAcls(String bucketName, String[] acls) throws OzoneException {
- HttpPut putRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(bucketName);
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
- putRequest = client.getHttpPut(builder.toString());
-
- for (String acl : acls) {
- putRequest
- .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl);
- }
- executePutBucket(putRequest, httpClient);
- } catch (URISyntaxException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Removes ACLs from a bucket.
- *
- * @param bucketName - Bucket Name
- * @param acls - Acls to be removed
- *
- * @throws OzoneException
- */
- public void removeAcls(String bucketName, String[] acls)
- throws OzoneException {
- HttpPut putRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(bucketName);
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
- putRequest = client.getHttpPut(builder.toString());
-
- for (String acl : acls) {
- putRequest
- .addHeader(Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl);
- }
- executePutBucket(putRequest, httpClient);
- } catch (URISyntaxException | IOException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(putRequest);
- }
- }
-
- /**
- * Returns information about an existing bucket.
- *
- * @param bucketName - BucketName
- *
- * @return OZoneBucket
- */
- public OzoneBucket getBucket(String bucketName) throws OzoneException {
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(bucketName);
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName() + "/" + bucketName)
- .setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_BUCKET).build();
- getRequest = client.getHttpGet(builder.toString());
- return executeInfoBucket(getRequest, httpClient);
-
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
-
- /**
- * Execute the info bucket call.
- *
- * @param getRequest - httpGet Request
- * @param httpClient - Http Client
- *
- * @return OzoneBucket
- *
- * @throws IOException
- * @throws OzoneException
- */
- private OzoneBucket executeInfoBucket(HttpGet getRequest,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(getRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- OzoneBucket bucket =
- new OzoneBucket(BucketInfo.parse(EntityUtils.toString(entity)),
- this);
- return bucket;
- }
- throw OzoneException.parse(EntityUtils.toString(entity));
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Execute the put bucket call.
- *
- * @param putRequest - http put request
- * @param httpClient - Http Client
- *
- * @return OzoneBucket
- *
- * @throws IOException
- * @throws OzoneException
- */
- private void executePutBucket(HttpPut putRequest,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(putRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (errorCode == HTTP_OK) {
- return;
- }
-
- if (entity != null) {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
-
- throw new OzoneRestClientException("Unexpected null in http result");
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Gets a list of buckets on this volume.
- *
- * @return - List of buckets
- *
- * @throws OzoneException
- */
- public List<OzoneBucket> listBuckets(String resultLength,
- String previousBucket, String prefix) throws OzoneException {
- HttpGet getRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName()).build();
- if (!Strings.isNullOrEmpty(resultLength)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_MAXKEYS, resultLength);
- }
- if (!Strings.isNullOrEmpty(previousBucket)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREVKEY, previousBucket);
- }
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.addParameter(Header.OZONE_LIST_QUERY_PREFIX, prefix);
- }
-
- getRequest = client.getHttpGet(builder.toString());
- return executeListBuckets(getRequest, httpClient);
-
- } catch (IOException | URISyntaxException e) {
- throw new OzoneRestClientException(e.getMessage(), e);
- } finally {
- OzoneClientUtils.releaseConnection(getRequest);
- }
- }
-
- /**
- * executes the List Bucket Call.
- *
- * @param getRequest - http Request
- * @param httpClient - http Client
- *
- * @return List of OzoneBuckets
- *
- * @throws IOException
- * @throws OzoneException
- */
- private List<OzoneBucket> executeListBuckets(HttpGet getRequest,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- List<OzoneBucket> ozoneBucketList = new LinkedList<OzoneBucket>();
- try {
- HttpResponse response = httpClient.execute(getRequest);
- int errorCode = response.getStatusLine().getStatusCode();
-
- entity = response.getEntity();
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload");
- }
- if (errorCode == HTTP_OK) {
- ListBuckets bucketList =
- ListBuckets.parse(EntityUtils.toString(entity));
-
- for (BucketInfo info : bucketList.getBuckets()) {
- ozoneBucketList.add(new OzoneBucket(info, this));
- }
- return ozoneBucketList;
-
- } else {
- throw OzoneException.parse(EntityUtils.toString(entity));
- }
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- /**
- * Delete an empty bucket.
- *
- * @param bucketName - Name of the bucket to delete
- *
- * @throws OzoneException
- */
- public void deleteBucket(String bucketName) throws OzoneException {
- HttpDelete delRequest = null;
- try (CloseableHttpClient httpClient = newHttpClient()) {
- OzoneUtils.verifyResourceName(bucketName);
- URIBuilder builder = new URIBuilder(getClient().getEndPointURI());
- builder.setPath("/" + getVolumeName() + "/" + bucketName).build();
-
- delRequest = client.getHttpDelete(builder.toString());
- executeDeleteBucket(delRequest, httpClient);
-
- } catch (IOException | URISyntaxException | IllegalArgumentException ex) {
- throw new OzoneRestClientException(ex.getMessage(), ex);
- } finally {
- OzoneClientUtils.releaseConnection(delRequest);
- }
- }
-
- /**
- * Executes delete bucket call.
- *
- * @param delRequest - Delete Request
- * @param httpClient - Http Client
-7 *
- * @throws IOException
- * @throws OzoneException
- */
- private void executeDeleteBucket(HttpDelete delRequest,
- CloseableHttpClient httpClient)
- throws IOException, OzoneException {
- HttpEntity entity = null;
- try {
- HttpResponse response = httpClient.execute(delRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- entity = response.getEntity();
-
- if (errorCode == HTTP_OK) {
- return;
- }
-
- if (entity == null) {
- throw new OzoneRestClientException("Unexpected null in http payload.");
- }
-
- throw OzoneException.parse(EntityUtils.toString(entity));
-
- } finally {
- if (entity != null) {
- EntityUtils.consumeQuietly(entity);
- }
- }
- }
-
- @VisibleForTesting
- public CloseableHttpClient newHttpClient() {
- return OzoneClientUtils.newHttpClient();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
deleted file mode 100644
index 046568b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/client/package-info.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Ozone client library is a java client for the Ozone
- * Object Store.
- */
-package org.apache.hadoop.ozone.web.client;
-
-/**
- This library is a simple Ozone REST Library.
-
- This library is a very *minimal* client written for tests and
- command line utils that work against Ozone. It does not have
- things like thread-pools and support for extended security models yet.
-
- OzoneClients return OzoneVolumes and OzoneVolumes return OzoneBuckets.
- **/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
deleted file mode 100644
index 16892e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.exceptions;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-
-import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
-import static java.net.HttpURLConnection.HTTP_CONFLICT;
-import static java.net.HttpURLConnection.HTTP_FORBIDDEN;
-import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
-import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
-import static java.net.HttpURLConnection.HTTP_UNAUTHORIZED;
-
-/**
- * Error Table represents the Errors from Ozone Rest API layer.
- *
- * Please note : The errors in this table are sorted by the HTTP_ERROR codes
- * if you add new error codes to this table please follow the same convention.
- */
-@InterfaceAudience.Private
-public final class ErrorTable {
-
- /* Error 400 */
- public static final OzoneException MISSING_VERSION =
- new OzoneException(HTTP_BAD_REQUEST, "missingVersion",
- "x-ozone-version header is required.");
-
- public static final OzoneException MISSING_DATE =
- new OzoneException(HTTP_BAD_REQUEST, "missingDate",
- "Date header is required.");
-
- public static final OzoneException BAD_DATE =
- new OzoneException(HTTP_BAD_REQUEST, "badDate",
- "Unable to parse date format.");
-
- public static final OzoneException MALFORMED_QUOTA =
- new OzoneException(HTTP_BAD_REQUEST, "malformedQuota",
- "Invalid quota specified.");
-
- public static final OzoneException MALFORMED_ACL =
- new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
- "Invalid ACL specified.");
-
-
- public static final OzoneException INVALID_VOLUME_NAME =
- new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
- "Invalid volume name.");
-
- public static final OzoneException INVALID_QUERY_PARAM =
- new OzoneException(HTTP_BAD_REQUEST, "invalidQueryParam",
- "Invalid query parameter.");
-
- public static final OzoneException INVALID_RESOURCE_NAME =
- new OzoneException(HTTP_BAD_REQUEST, "invalidResourceName",
- "Invalid volume, bucket or key name.");
-
- public static final OzoneException INVALID_BUCKET_NAME =
- new OzoneException(HTTP_BAD_REQUEST, "invalidBucketName",
- "Invalid bucket name.");
-
- public static final OzoneException INVALID_KEY =
- new OzoneException(HTTP_BAD_REQUEST, "invalidKey", "Invalid key.");
-
- public static final OzoneException INVALID_REQUEST =
- new OzoneException(HTTP_BAD_REQUEST, "invalidRequest",
- "Error in request.");
-
- public static final OzoneException MALFORMED_BUCKET_VERSION =
- new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
- "Malformed bucket version or version not unique.");
-
- public static final OzoneException MALFORMED_STORAGE_TYPE =
- new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
- "Invalid storage Type specified.");
-
- public static final OzoneException MALFORMED_STORAGE_CLASS =
- new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
- "Invalid storage class specified.");
-
- public static final OzoneException BAD_DIGEST =
- new OzoneException(HTTP_BAD_REQUEST, "badDigest",
- "Content MD5 does not match.");
-
- public static final OzoneException INCOMPLETE_BODY =
- new OzoneException(HTTP_BAD_REQUEST, "incompleteBody",
- "Content length does not match stream size.");
-
- public static final OzoneException BAD_AUTHORIZATION =
- new OzoneException(HTTP_BAD_REQUEST, "badAuthorization",
- "Missing authorization or authorization has to be " +
- "unique.");
-
- public static final OzoneException BAD_PROPERTY =
- new OzoneException(HTTP_BAD_REQUEST, "unknownProperty",
- "This property is not supported by this server.");
-
- /* Error 401 */
- public static final OzoneException UNAUTHORIZED =
- new OzoneException(HTTP_UNAUTHORIZED, "Unauthorized",
- "Access token is missing or invalid token.");
-
- /* Error 403 */
- public static final OzoneException ACCESS_DENIED =
- new OzoneException(HTTP_FORBIDDEN, "accessDenied", "Access denied.");
-
- /* Error 404 */
- public static final OzoneException USER_NOT_FOUND =
- new OzoneException(HTTP_NOT_FOUND, "userNotFound", "Invalid user name.");
-
- public static final OzoneException VOLUME_NOT_FOUND =
- new OzoneException(HTTP_NOT_FOUND, "volumeNotFound", "No such volume.");
-
- /* Error 409 */
- public static final OzoneException VOLUME_ALREADY_EXISTS =
- new OzoneException(HTTP_CONFLICT, "volumeAlreadyExists",
- "Duplicate volume name.");
-
- public static final OzoneException BUCKET_ALREADY_EXISTS =
- new OzoneException(HTTP_CONFLICT, "bucketAlreadyExists",
- "Duplicate bucket name.");
-
- public static final OzoneException VOLUME_NOT_EMPTY =
- new OzoneException(HTTP_CONFLICT, "volumeNotEmpty",
- "Volume must not have any buckets.");
-
- public static final OzoneException BUCKET_NOT_EMPTY =
- new OzoneException(HTTP_CONFLICT, "bucketNotEmpty",
- "Bucket must not have any keys.");
-
- public static final OzoneException KEY_OPERATION_CONFLICT =
- new OzoneException(HTTP_CONFLICT, "keyOperationConflict",
- "Conflicting operation on the specified key is going" +
- " on.");
-
- /* Error 500 */
- public static final OzoneException SERVER_ERROR =
- new OzoneException(HTTP_INTERNAL_ERROR, "internalServerError",
- "Internal server error.");
-
- /**
- * Create a new instance of Error.
- *
- * @param e Error Template
- * @param requestID Request ID
- * @param resource Resource Name
- * @param hostID hostID
- *
- * @return creates a new instance of error based on the template
- */
- public static OzoneException newError(OzoneException e, String requestID,
- String resource, String hostID) {
- OzoneException err =
- new OzoneException(e.getHttpCode(), e.getShortMessage(),
- e.getMessage());
- err.setRequestId(requestID);
- err.setResource(resource);
- err.setHostID(hostID);
- return err;
- }
-
- /**
- * Create new instance of Error.
- *
- * @param e - Error Template
- * @param args - Args
- *
- * @return Ozone Exception
- */
- public static OzoneException newError(OzoneException e, UserArgs args) {
- OzoneException err =
- new OzoneException(e.getHttpCode(), e.getShortMessage(),
- e.getMessage());
- if (args != null) {
- err.setRequestId(args.getRequestID());
- err.setResource(args.getResourceName());
- err.setHostID(args.getHostName());
- }
- return err;
- }
-
- /**
- * Create new instance of Error.
- *
- * @param e - Error Template
- * @param args - Args
- * @param ex Exception
- *
- * @return Ozone Exception
- */
- public static OzoneException newError(OzoneException e, UserArgs args,
- Exception ex) {
- OzoneException err =
- new OzoneException(e.getHttpCode(), e.getShortMessage(), ex);
-
- if(args != null) {
- err.setRequestId(args.getRequestID());
- err.setResource(args.getResourceName());
- err.setHostID(args.getHostName());
- }
- err.setMessage(ex.getMessage());
- return err;
- }
-
- private ErrorTable() {
- // Never constructed.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
deleted file mode 100644
index 59cf724..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.exceptions;
-
-/**
- This package contains ozone client side libraries.
- */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
deleted file mode 100644
index 0d5248d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * BucketArgs packages all bucket related arguments to
- * file system calls.
- */
-public class BucketArgs extends VolumeArgs {
- private final String bucketName;
- private List<OzoneAcl> addAcls;
- private List<OzoneAcl> removeAcls;
- private OzoneConsts.Versioning versioning;
- private StorageType storageType;
-
- /**
- * Constructor for BucketArgs.
- *
- * @param volumeName - volumeName
- * @param bucketName - bucket Name
- * @param userArgs - userArgs
- */
- public BucketArgs(String volumeName, String bucketName, UserArgs userArgs) {
- super(volumeName, userArgs);
- this.bucketName = bucketName;
- this.versioning = OzoneConsts.Versioning.NOT_DEFINED;
- this.storageType = null;
- }
-
-
- /**
- * Constructor for BucketArgs.
- *
- * @param bucketName - bucket Name
- * @param volumeArgs - volume Args
- */
- public BucketArgs(String bucketName, VolumeArgs volumeArgs) {
- super(volumeArgs);
- this.bucketName = bucketName;
- this.versioning = OzoneConsts.Versioning.NOT_DEFINED;
- this.storageType = null;
- }
-
- /**
- * Constructor for BucketArgs.
- *
- * @param args - Bucket Args
- */
- public BucketArgs(BucketArgs args) {
- this(args.getBucketName(), args);
- this.setAddAcls(args.getAddAcls());
- this.setRemoveAcls(args.getRemoveAcls());
- }
-
- /**
- * Returns the Bucket Name.
- *
- * @return Bucket Name
- */
- public String getBucketName() {
- return bucketName;
- }
-
- /**
- * Returns Additive ACLs for the Bucket if specified.
- *
- * @return acls
- */
- public List<OzoneAcl> getAddAcls() {
- return addAcls;
- }
-
- /**
- * Set Additive ACLs.
- *
- * @param acl - ACL
- */
- public void setAddAcls(List<OzoneAcl> acl) {
- this.addAcls = acl;
- }
-
- /**
- * Returns remove ACLs for the Bucket if specified.
- *
- * @return acls
- */
- public List<OzoneAcl> getRemoveAcls() {
- return removeAcls;
- }
-
- /**
- * Takes an ACL and sets the ACL object to ACL represented by the String.
- *
- * @param aclString - aclString
- */
- public void addAcls(List<String> aclString) throws IllegalArgumentException {
- if (aclString == null) {
- throw new IllegalArgumentException("ACLs cannot be null");
- }
- if (this.addAcls == null) {
- this.addAcls = new LinkedList<>();
- }
- for (String s : aclString) {
- this.addAcls.add(OzoneAcl.parseAcl(s));
- }
- }
-
- /**
- * Takes an ACL and sets the ACL object to ACL represented by the String.
- *
- * @param aclString - aclString
- */
- public void removeAcls(List<String> aclString)
- throws IllegalArgumentException {
- if (aclString == null) {
- throw new IllegalArgumentException("ACLs cannot be null");
- }
- if (this.removeAcls == null) {
- this.removeAcls = new LinkedList<>();
- }
- for (String s : aclString) {
- this.removeAcls.add(OzoneAcl.parseAcl(s));
- }
- }
-
- /**
- * Set remove ACLs.
- *
- * @param acl - ACL
- */
- public void setRemoveAcls(List<OzoneAcl> acl) {
- this.removeAcls = acl;
- }
-
-
- /**
- * Returns Versioning Info.
- *
- * @return versioning
- */
- public OzoneConsts.Versioning getVersioning() {
- return versioning;
- }
-
-
- /**
- * SetVersioning Info.
- *
- * @param versioning - Enum value
- */
- public void setVersioning(OzoneConsts.Versioning versioning) {
- this.versioning = versioning;
- }
-
- /**
- * returns the current Storage Class.
- *
- * @return Storage Class
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Sets the Storage Class.
- *
- * @param storageType Set Storage Class
- */
- public void setStorageType(StorageType storageType) {
- this.storageType = storageType;
- }
-
- /**
- * returns - Volume/bucketName.
- *
- * @return String
- */
- @Override
- public String getResourceName() {
- return getVolumeName() + "/" + getBucketName();
- }
-
- /**
- * Returns User/Volume name which is the parent of this
- * bucket.
- *
- * @return String
- */
- public String getParentName() {
- return getUserName() + "/" + getVolumeName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
deleted file mode 100644
index d090499..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/*
- *
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.Bucket;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
-
-
-/**
- * Bucket Class handles all ozone Bucket related actions.
- */
-public class BucketHandler implements Bucket {
- /**
- * createBucket call handles the POST request for Creating a Bucket.
- *
- * @param volume - Volume name
- * @param bucket - Bucket Name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @Override
- public Response createBucket(String volume, String bucket, Request req,
- UriInfo info, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "createBucket");
- return new BucketProcessTemplate() {
- @Override
- public Response doProcess(BucketArgs args)
- throws OzoneException, IOException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- getAclsFromHeaders(args, false);
- args.setVersioning(getVersioning(args));
- args.setStorageType(getStorageType(args));
- fs.createBucket(args);
- return OzoneUtils.getResponse(args, HTTP_CREATED, "");
- }
- }.handleCall(volume, bucket, req, info, headers);
- }
-
- /**
- * updateBucket call handles the PUT request for updating a Bucket.
- *
- * There are only three possible actions currently with updateBucket.
- * They are add/remove on ACLS, Bucket Versioning and StorageType.
- * if you make a call with any other action, update just returns 200 OK.
- *
- * @param volume - Storage volume name
- * @param bucket - Bucket name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @Override
- public Response updateBucket(String volume, String bucket, Request req,
- UriInfo info, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "updateBucket");
- return new BucketProcessTemplate() {
- @Override
- public Response doProcess(BucketArgs args)
- throws OzoneException, IOException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- getAclsFromHeaders(args, true);
- args.setVersioning(getVersioning(args));
- args.setStorageType(getStorageType(args));
-
- if ((args.getAddAcls() != null) || (args.getRemoveAcls() != null)) {
- fs.setBucketAcls(args);
- }
-
- if (args.getVersioning() != OzoneConsts.Versioning.NOT_DEFINED) {
- fs.setBucketVersioning(args);
- }
-
- if (args.getStorageType() != null) {
- fs.setBucketStorageClass(args);
- }
- return OzoneUtils.getResponse(args, HTTP_OK, "");
- }
- }.handleCall(volume, bucket, req, info, headers);
- }
-
- /**
- * Deletes an empty bucket.
- *
- * @param volume Volume name
- * @param bucket Bucket Name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @Override
- public Response deleteBucket(String volume, String bucket, Request req,
- UriInfo info, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "deleteBucket");
- return new BucketProcessTemplate() {
- @Override
- public Response doProcess(BucketArgs args)
- throws OzoneException, IOException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- fs.deleteBucket(args);
- return OzoneUtils.getResponse(args, HTTP_OK, "");
- }
- }.handleCall(volume, bucket, req, info, headers);
- }
-
- /**
- * List Buckets allows the user to list the bucket.
- *
- * @param volume - Storage Volume Name
- * @param bucket - Bucket Name
- * @param info - Uri Info
- * @param prefix - Prefix for the keys to be fetched
- * @param maxKeys - MaxNumber of Keys to Return
- * @param startPage - Continuation Token
- * @param req - Http request
- * @param headers - Http headers
- *
- * @return - Json Body
- *
- * @throws OzoneException
- */
- @Override
- public Response listBucket(String volume, String bucket, final String info,
- final String prefix, final int maxKeys,
- final String startPage, Request req,
- UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "listBucket");
- return new BucketProcessTemplate() {
- @Override
- public Response doProcess(BucketArgs args)
- throws OzoneException, IOException {
- switch (info) {
- case Header.OZONE_INFO_QUERY_KEY:
- ListArgs listArgs = new ListArgs(args, prefix, maxKeys, startPage);
- return getBucketKeysList(listArgs);
- case Header.OZONE_INFO_QUERY_BUCKET:
- return getBucketInfoResponse(args);
- default:
- OzoneException ozException =
- ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
- ozException.setMessage("Unrecognized query param : " + info);
- throw ozException;
- }
- }
- }.handleCall(volume, bucket, req, uriInfo, headers);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
deleted file mode 100644
index 4736746..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketProcessTemplate.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.NoSuchFileException;
-import java.util.LinkedList;
-import java.util.List;
-
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-/**
- * This class abstracts way the repetitive tasks in
- * Bucket handling code.
- */
-public abstract class BucketProcessTemplate {
- private static final Logger LOG =
- LoggerFactory.getLogger(BucketProcessTemplate.class);
-
- /**
- * This function serves as the common error handling function
- * for all bucket related operations.
- *
- * @param volume - Volume Name
- * @param bucket - Bucket Name
- * @param request - Http Request
- * @param uriInfo - Http Uri
- * @param headers - Http Headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- public Response handleCall(String volume, String bucket, Request request,
- UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
- // TODO : Add logging
- String reqID = OzoneUtils.getRequestID();
- String hostName = OzoneUtils.getHostName();
- MDC.put(OZONE_COMPONENT, "ozone");
- MDC.put(OZONE_REQUEST, reqID);
- UserArgs userArgs = null;
- try {
- userArgs = new UserArgs(reqID, hostName, request, uriInfo, headers);
-
- OzoneUtils.validate(request, headers, reqID, bucket, hostName);
- OzoneUtils.verifyResourceName(bucket);
-
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- userArgs.setUserName(auth.getUser(userArgs));
- MDC.put(OZONE_USER, userArgs.getUserName());
-
- BucketArgs args = new BucketArgs(volume, bucket, userArgs);
- MDC.put(OZONE_RESOURCE, args.getResourceName());
- Response response = doProcess(args);
- LOG.debug("Success");
- MDC.clear();
- return response;
-
- } catch (IllegalArgumentException argEx) {
- LOG.error("Invalid bucket.", argEx);
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, userArgs,
- argEx);
- } catch (IOException fsExp) {
- handleIOException(bucket, reqID, hostName, fsExp);
- }
- return null;
- }
-
- /**
- * Reads ACLs from headers and throws appropriate exception if needed.
- *
- * @param args - bucketArgs
- *
- * @throws OzoneException
- */
- void getAclsFromHeaders(BucketArgs args, boolean parseRemoveACL)
- throws OzoneException {
- try {
- List<String> acls = getAcls(args, Header.OZONE_ACL_REMOVE);
- if (acls != null && !acls.isEmpty()) {
- args.removeAcls(acls);
- }
- if ((!parseRemoveACL) && args.getRemoveAcls() != null) {
- OzoneException ex = ErrorTable.newError(ErrorTable.MALFORMED_ACL, args);
- ex.setMessage("Invalid Remove ACLs");
- throw ex;
- }
-
- acls = getAcls(args, Header.OZONE_ACL_ADD);
- if (acls != null && !acls.isEmpty()) {
- args.addAcls(acls);
- }
- } catch (IllegalArgumentException ex) {
- throw ErrorTable.newError(ErrorTable.MALFORMED_ACL, args, ex);
- }
- }
-
- /**
- * Converts FileSystem IO exceptions to OZONE exceptions.
- *
- * @param bucket Name of the bucket
- * @param reqID Request ID
- * @param hostName Machine Name
- * @param fsExp Exception
- *
- * @throws OzoneException
- */
- void handleIOException(String bucket, String reqID, String hostName,
- IOException fsExp) throws OzoneException {
- LOG.error("IOException:", fsExp);
-
- OzoneException exp = null;
- if (fsExp instanceof FileAlreadyExistsException) {
- exp = ErrorTable
- .newError(ErrorTable.BUCKET_ALREADY_EXISTS, reqID, bucket, hostName);
- }
-
- if (fsExp instanceof DirectoryNotEmptyException) {
- exp = ErrorTable
- .newError(ErrorTable.BUCKET_NOT_EMPTY, reqID, bucket, hostName);
- }
-
- if (fsExp instanceof NoSuchFileException) {
- exp = ErrorTable
- .newError(ErrorTable.INVALID_BUCKET_NAME, reqID, bucket, hostName);
- }
-
- // Default we don't handle this exception yet,
- // report a Server Internal Error.
- if (exp == null) {
- exp =
- ErrorTable.newError(ErrorTable.SERVER_ERROR, reqID, bucket, hostName);
- if (fsExp != null) {
- exp.setMessage(fsExp.getMessage());
- }
- }
- throw exp;
- }
-
- /**
- * Abstract function that gets implemented in the BucketHandler functions.
- * This function will just deal with the core file system related logic
- * and will rely on handleCall function for repetitive error checks
- *
- * @param args - parsed bucket args, name, userName, ACLs etc
- *
- * @return Response
- *
- * @throws OzoneException
- * @throws IOException
- */
- public abstract Response doProcess(BucketArgs args)
- throws OzoneException, IOException;
-
-
- /**
- * Returns the ACL String if available.
- * This function ignores all ACLs that are not prefixed with either
- * ADD or Remove
- *
- * @param args - BucketArgs
- * @param tag - Tag for different type of acls
- *
- * @return List of ACLs
- *
- */
- List<String> getAcls(BucketArgs args, String tag) {
- List<String> aclStrings =
- args.getHeaders().getRequestHeader(Header.OZONE_ACLS);
- List<String> filteredSet = null;
- if (aclStrings != null) {
- filteredSet = new LinkedList<>();
- for (String s : aclStrings) {
- if (s.startsWith(tag)) {
- filteredSet.add(s.replaceFirst(tag, ""));
- }
- }
- }
- return filteredSet;
- }
-
- /**
- * Returns bucket versioning Info.
- *
- * @param args - BucketArgs
- *
- * @return - String
- *
- * @throws OzoneException
- */
- OzoneConsts.Versioning getVersioning(BucketArgs args) throws OzoneException {
-
- List<String> versionStrings =
- args.getHeaders().getRequestHeader(Header.OZONE_BUCKET_VERSIONING);
- if (versionStrings == null) {
- return null;
- }
-
- if (versionStrings.size() > 1) {
- OzoneException ex =
- ErrorTable.newError(ErrorTable.MALFORMED_BUCKET_VERSION, args);
- ex.setMessage("Exactly one bucket version header required");
- throw ex;
- }
-
- String version = versionStrings.get(0);
- try {
- return OzoneConsts.Versioning.valueOf(version);
- } catch (IllegalArgumentException ex) {
- LOG.debug("Malformed Version. version: {}", version);
- throw ErrorTable.newError(ErrorTable.MALFORMED_BUCKET_VERSION, args, ex);
- }
- }
-
-
- /**
- * Returns Storage Class if Available or returns Default.
- *
- * @param args - bucketArgs
- *
- * @return StorageType
- *
- * @throws OzoneException
- */
- StorageType getStorageType(BucketArgs args) throws OzoneException {
- List<String> storageClassString = null;
- try {
- storageClassString =
- args.getHeaders().getRequestHeader(Header.OZONE_STORAGE_TYPE);
-
- if (storageClassString == null) {
- return null;
- }
- if (storageClassString.size() > 1) {
- OzoneException ex =
- ErrorTable.newError(ErrorTable.MALFORMED_STORAGE_TYPE, args);
- ex.setMessage("Exactly one storage class header required");
- throw ex;
- }
- return StorageType.valueOf(storageClassString.get(0).toUpperCase());
- } catch (IllegalArgumentException ex) {
- if(storageClassString != null) {
- LOG.debug("Malformed storage type. Type: {}",
- storageClassString.get(0).toUpperCase());
- }
- throw ErrorTable.newError(ErrorTable.MALFORMED_STORAGE_TYPE, args, ex);
- }
- }
-
- /**
- * Returns BucketInfo response.
- *
- * @param args - BucketArgs
- *
- * @return BucketInfo
- *
- * @throws IOException
- * @throws OzoneException
- */
- Response getBucketInfoResponse(BucketArgs args)
- throws IOException, OzoneException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- BucketInfo info = fs.getBucketInfo(args);
- return OzoneUtils.getResponse(args, HTTP_OK, info.toJsonString());
- }
-
- /**
- * Returns list of objects in a bucket.
- * @param args - ListArgs
- * @return Response
- * @throws IOException
- * @throws OzoneException
- */
- Response getBucketKeysList(ListArgs args) throws IOException, OzoneException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- ListKeys objects = fs.listKeys(args);
- return OzoneUtils.getResponse(args.getArgs(), HTTP_OK,
- objects.toJsonString());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
deleted file mode 100644
index 48a4cb4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyArgs.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- * Class that packages all key Arguments.
- */
-public class KeyArgs extends BucketArgs {
- private String key;
- private String hash;
- private long size;
-
- /**
- * Constructor for Key Args.
- *
- * @param volumeName - Volume Name
- * @param bucketName - Bucket Name
- * @param objectName - Key
- */
- public KeyArgs(String volumeName, String bucketName,
- String objectName, UserArgs args) {
- super(volumeName, bucketName, args);
- this.key = objectName;
- }
-
- /**
- * Constructor for Key Args.
- *
- * @param objectName - Key
- * @param args - Bucket Args
- */
- public KeyArgs(String objectName, BucketArgs args) {
- super(args);
- this.key = objectName;
- }
-
- /**
- * Get Key Name.
- *
- * @return String
- */
- public String getKeyName() {
- return this.key;
- }
-
- /**
- * Computed File hash.
- *
- * @return String
- */
- public String getHash() {
- return hash;
- }
-
- /**
- * Sets the hash String.
- *
- * @param hash String
- */
- public void setHash(String hash) {
- this.hash = hash;
- }
-
- /**
- * Returns the file size.
- *
- * @return long - file size
- */
- public long getSize() {
- return size;
- }
-
- /**
- * Set Size.
- *
- * @param size Size of the file
- */
- public void setSize(long size) {
- this.size = size;
- }
-
- /**
- * Returns the name of the resource.
- *
- * @return String
- */
- @Override
- public String getResourceName() {
- return super.getResourceName() + "/" + getKeyName();
- }
-
- /**
- * Parent name of this resource.
- *
- * @return String.
- */
- @Override
- public String getParentName() {
- return super.getResourceName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
deleted file mode 100644
index eff2759..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyHandler.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.commons.codec.binary.Hex;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.Keys;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * KeyHandler deals with basic Key Operations.
- */
-public class KeyHandler implements Keys {
-
- /**
- * Gets the Key/key information if it exists.
- *
- * @param volume Storage Volume
- * @param bucket Name of the bucket
- * @param key Name of the key
- * @param info Tag info
- * @param req Request
- * @param uriInfo Uri Info
- * @param headers Http Header
- * @return Response
- * @throws OzoneException
- */
- @Override
- public Response getKey(String volume, String bucket, String key, String info,
- Request req, UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
- return new KeyProcessTemplate() {
- /**
- * Abstract function that gets implemented in the KeyHandler functions.
- * This function will just deal with the core file system related logic
- * and will rely on handleCall function for repetitive error checks
- *
- * @param args - parsed bucket args, name, userName, ACLs etc
- * @param input - The body as an Input Stream
- * @param request - Http request
- * @param headers - Parsed http Headers.
- * @param uriInfo - UriInfo
- *
- * @return Response
- *
- * @throws IOException - From the file system operations
- */
- @Override
- public Response doProcess(KeyArgs args, InputStream input,
- Request request, HttpHeaders headers,
- UriInfo uriInfo)
- throws IOException, OzoneException, NoSuchAlgorithmException {
- if (info == null) {
- return getKey(args);
- } else if (info.equals(Header.OZONE_INFO_QUERY_KEY)) {
- return getKeyInfo(args);
- }
-
- OzoneException ozException = ErrorTable
- .newError(ErrorTable.INVALID_QUERY_PARAM, args);
- ozException.setMessage("Unrecognized query param : " + info);
- throw ozException;
- }
- }.handleCall(volume, bucket, key, req, headers, uriInfo, null);
- }
-
- /**
- * Gets the Key if it exists.
- */
- private Response getKey(KeyArgs args)
- throws IOException, OzoneException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- LengthInputStream stream = fs.newKeyReader(args);
- return OzoneUtils.getResponse(args, HTTP_OK, stream);
- }
-
- /**
- * Gets the Key information if it exists.
- */
- private Response getKeyInfo(KeyArgs args)
- throws IOException, OzoneException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- KeyInfo keyInfo = fs.getKeyInfo(args);
- return OzoneUtils.getResponse(args, HTTP_OK, keyInfo.toJsonString());
- }
-
- /**
- * Adds a key to an existing bucket. If the object already exists this call
- * will overwrite or add with new version number if the bucket versioning is
- * turned on.
- *
- * @param volume Storage Volume Name
- * @param bucket Name of the bucket
- * @param keys Name of the Object
- * @param is InputStream or File Data
- * @param req Request
- * @param info - UriInfo
- * @param headers http headers
- * @return Response
- * @throws OzoneException
- */
- @Override
- public Response putKey(String volume, String bucket, String keys,
- InputStream is, Request req, UriInfo info,
- HttpHeaders headers) throws OzoneException {
-
- return new KeyProcessTemplate() {
- /**
- * Abstract function that gets implemented in the KeyHandler functions.
- * This function will just deal with the core file system related logic
- * and will rely on handleCall function for repetitive error checks
- *
- * @param args - parsed bucket args, name, userName, ACLs etc
- * @param input - The body as an Input Stream
- * @param request - Http request
- * @param headers - Parsed http Headers.
- * @param info - UriInfo
- *
- * @return Response
- *
- * @throws IOException - From the file system operations
- */
- @Override
- public Response doProcess(KeyArgs args, InputStream input,
- Request request, HttpHeaders headers,
- UriInfo info)
- throws IOException, OzoneException, NoSuchAlgorithmException {
- final int eof = -1;
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
-
- byte[] buffer = new byte[4 * 1024];
- String contentLenString = getContentLength(headers, args);
- String newLen = contentLenString.replaceAll("\"", "");
- int contentLen = Integer.parseInt(newLen);
- args.setSize(contentLen);
-
- MessageDigest md5 = MessageDigest.getInstance("MD5");
- int bytesRead = 0;
- int len = 0;
- OutputStream stream = fs.newKeyWriter(args);
- while ((bytesRead < contentLen) && (len != eof)) {
- int readSize =
- (contentLen - bytesRead > buffer.length) ? buffer.length :
- contentLen - bytesRead;
- len = input.read(buffer, 0, readSize);
- if (len != eof) {
- stream.write(buffer, 0, len);
- md5.update(buffer, 0, len);
- bytesRead += len;
-
- }
- }
-
- checkFileLengthMatch(args, fs, contentLen, bytesRead);
-
- String hashString = Hex.encodeHexString(md5.digest());
-// TODO : Enable hash value checking.
-// String contentHash = getContentMD5(headers, args);
-// checkFileHashMatch(args, hashString, fs, contentHash);
- args.setHash(hashString);
- args.setSize(bytesRead);
- fs.commitKey(args, stream);
- return OzoneUtils.getResponse(args, HTTP_CREATED, "");
- }
- }.handleCall(volume, bucket, keys, req, headers, info, is);
- }
-
- /**
- * Deletes an existing key.
- *
- * @param volume Storage Volume Name
- * @param bucket Name of the bucket
- * @param keys Name of the Object
- * @param req http Request
- * @param info - UriInfo
- * @param headers HttpHeaders
- * @return Response
- * @throws OzoneException
- */
- @Override
- public Response deleteKey(String volume, String bucket, String keys,
- Request req, UriInfo info, HttpHeaders headers)
- throws OzoneException {
- return new KeyProcessTemplate() {
- /**
- * Abstract function that gets implemented in the KeyHandler functions.
- * This function will just deal with the core file system related logic
- * and will rely on handleCall function for repetitive error checks
- *
- * @param args - parsed bucket args, name, userName, ACLs etc
- * @param input - The body as an Input Stream
- * @param request - Http request
- * @param headers - Parsed http Headers.
- * @param info - UriInfo
- *
- * @return Response
- *
- * @throws IOException - From the file system operations
- */
- @Override
- public Response doProcess(KeyArgs args, InputStream input,
- Request request, HttpHeaders headers,
- UriInfo info)
- throws IOException, OzoneException, NoSuchAlgorithmException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- fs.deleteKey(args);
- return OzoneUtils.getResponse(args, HTTP_OK, "");
- }
- }.handleCall(volume, bucket, keys, req, headers, info, null);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
deleted file mode 100644
index dcf0fc5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/KeyProcessTemplate.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.BAD_DIGEST;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INCOMPLETE_BODY;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_BUCKET_NAME;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.INVALID_REQUEST;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.SERVER_ERROR;
-import static org.apache.hadoop.ozone.web.exceptions.ErrorTable.newError;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-
-/**
- * This class abstracts way the repetitive tasks in Key handling code.
- */
-public abstract class KeyProcessTemplate {
- private static final Logger LOG =
- LoggerFactory.getLogger(KeyProcessTemplate.class);
-
- /**
- * This function serves as the common error handling function for all Key
- * related operations.
- *
- * @param bucket bucket Name
- * @param key the object name
- * @param headers Http headers
- * @param is Input XML stream
- * @throws OzoneException
- */
- public Response handleCall(String volume, String bucket, String key,
- Request request, HttpHeaders headers, UriInfo info,
- InputStream is) throws OzoneException {
-
- String reqID = OzoneUtils.getRequestID();
- String hostName = OzoneUtils.getHostName();
- MDC.put(OZONE_COMPONENT, "ozone");
- MDC.put(OZONE_REQUEST, reqID);
- UserArgs userArgs = null;
- try {
- userArgs = new UserArgs(reqID, hostName, request, info, headers);
- OzoneUtils.validate(request, headers, reqID, bucket, hostName);
- OzoneUtils.verifyResourceName(bucket);
-
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- userArgs.setUserName(auth.getUser(userArgs));
- MDC.put(OZONE_USER, userArgs.getUserName());
-
- KeyArgs args = new KeyArgs(volume, bucket, key, userArgs);
- MDC.put(OZONE_RESOURCE, args.getResourceName());
- Response response = doProcess(args, is, request, headers, info);
- LOG.debug("Success");
- MDC.clear();
- return response;
-
- } catch (IllegalArgumentException argExp) {
- LOG.error("Invalid bucket in key call.", argExp);
- throw newError(INVALID_BUCKET_NAME, userArgs, argExp);
- } catch (IOException fsExp) {
- LOG.error("IOException:", fsExp);
- // Map KEY_NOT_FOUND to INVALID_KEY
- if (fsExp.getMessage().endsWith(
- KeySpaceManagerProtocolProtos.Status.KEY_NOT_FOUND.name())) {
- throw ErrorTable.newError(ErrorTable.INVALID_KEY, userArgs, fsExp);
- }
-
- // TODO : Handle errors from the FileSystem , let us map to server error
- // for now.
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, userArgs, fsExp);
- } catch (NoSuchAlgorithmException algoEx) {
- LOG.error("NoSuchAlgorithmException. Probably indicates an unusual java "
- + "installation.", algoEx);
- throw ErrorTable.newError(SERVER_ERROR, userArgs, algoEx);
- }
- }
-
- /**
- * Abstract function that gets implemented in the KeyHandler functions. This
- * function will just deal with the core file system related logic and will
- * rely on handleCall function for repetitive error checks
- *
- * @param args - parsed bucket args, name, userName, ACLs etc
- * @param input - The body as an Input Stream
- * @param request - Http request
- * @param headers - Parsed http Headers.
- * @param info - UriInfo
- * @return Response
- * @throws IOException - From the file system operations
- */
- public abstract Response doProcess(KeyArgs args, InputStream input,
- Request request, HttpHeaders headers,
- UriInfo info)
- throws IOException, OzoneException, NoSuchAlgorithmException;
-
- /**
- * checks if the File Content-MD5 we wrote matches the hash we computed from
- * the stream. if it does match we delete the file and throw and exception to
- * let the user know that we have a hash mismatch
- *
- * @param args Object Args
- * @param computedString MD5 hash value
- * @param fs Pointer to File System so we can delete the file
- * @param contentHash User Specified hash string
- * @throws IOException
- * @throws OzoneException
- */
- public void checkFileHashMatch(KeyArgs args, String computedString,
- StorageHandler fs, String contentHash)
- throws IOException, OzoneException {
- if (contentHash != null) {
- String contentString =
- new String(Base64.decodeBase64(contentHash), OzoneUtils.ENCODING)
- .trim();
-
- if (!contentString.equals(computedString)) {
- fs.deleteKey(args);
- OzoneException ex = ErrorTable.newError(BAD_DIGEST, args);
- String msg = String.format("MD5 Digest mismatch. Expected %s Found " +
- "%s", contentString, computedString);
- ex.setMessage(msg);
- LOG.debug(msg);
- throw ex;
- }
- }
- }
-
- /**
- * check if the content-length matches the actual stream length. if we find a
- * mismatch we will delete the file and throw an exception to let the user
- * know that length mismatch detected
- *
- * @param args Object Args
- * @param fs Pointer to File System Object, to delete the file that we
- * wrote
- * @param contentLen Http Content-Length Header
- * @param bytesRead Actual Bytes we read from the stream
- * @throws IOException
- * @throws OzoneException
- */
- public void checkFileLengthMatch(KeyArgs args, StorageHandler fs,
- int contentLen, int bytesRead)
- throws IOException, OzoneException {
- if (bytesRead != contentLen) {
- fs.deleteKey(args);
- OzoneException ex = ErrorTable.newError(INCOMPLETE_BODY, args);
- String msg = String.format("Body length mismatch. Expected length : %d" +
- " Found %d", contentLen, bytesRead);
- ex.setMessage(msg);
- LOG.debug(msg);
- throw ex;
- }
- }
-
- /**
- * Returns Content Length header value if available.
- *
- * @param headers - Http Headers
- * @return - String or null
- */
- public String getContentLength(HttpHeaders headers, KeyArgs args)
- throws OzoneException {
- List<String> contentLengthList =
- headers.getRequestHeader(HttpHeaders.CONTENT_LENGTH);
- if ((contentLengthList != null) && (contentLengthList.size() > 0)) {
- return contentLengthList.get(0);
- }
-
- OzoneException ex = ErrorTable.newError(INVALID_REQUEST, args);
- ex.setMessage("Content-Length is a required header for putting a key.");
- throw ex;
-
- }
-
- /**
- * Returns Content MD5 value if available.
- *
- * @param headers - Http Headers
- * @return - String or null
- */
- public String getContentMD5(HttpHeaders headers, KeyArgs args) {
- List<String> contentLengthList =
- headers.getRequestHeader(Header.CONTENT_MD5);
- if ((contentLengthList != null) && (contentLengthList.size() > 0)) {
- return contentLengthList.get(0);
- }
-// TODO : Should we make this compulsory ?
-// OzoneException ex = ErrorTable.newError(ErrorTable.invalidRequest, args);
-// ex.setMessage("Content-MD5 is a required header for putting a key");
-// throw ex;
- return "";
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
deleted file mode 100644
index 49ca4a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ListArgs.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- * Supports listing keys with pagination.
- */
-public class ListArgs<T extends UserArgs> {
- private String prevKey;
- private String prefix;
- private int maxKeys;
- private boolean rootScan;
- private T args;
-
- /**
- * Constructor for ListArgs.
- *
- * @param args - BucketArgs
- * @param prefix Prefix to start Query from
- * @param maxKeys Max result set
- * @param prevKey - Page token
- */
- public ListArgs(T args, String prefix, int maxKeys,
- String prevKey) {
- setArgs(args);
- setPrefix(prefix);
- setMaxKeys(maxKeys);
- setPrevKey(prevKey);
- }
-
- /**
- * Copy Constructor for ListArgs.
- *
- * @param args - List Args
- */
- public ListArgs(T args, ListArgs listArgs) {
- this(args, listArgs.getPrefix(), listArgs.getMaxKeys(),
- listArgs.getPrevKey());
- }
-
- /**
- * Returns page token.
- *
- * @return String
- */
- public String getPrevKey() {
- return prevKey;
- }
-
- /**
- * Sets page token.
- *
- * @param prevKey - Page token
- */
- public void setPrevKey(String prevKey) {
- this.prevKey = prevKey;
- }
-
- /**
- * Gets max keys.
- *
- * @return int
- */
- public int getMaxKeys() {
- return maxKeys;
- }
-
- /**
- * Sets max keys.
- *
- * @param maxKeys - Maximum keys to return
- */
- public void setMaxKeys(int maxKeys) {
- this.maxKeys = maxKeys;
- }
-
- /**
- * Gets prefix.
- *
- * @return String
- */
- public String getPrefix() {
- return prefix;
- }
-
- /**
- * Sets prefix.
- *
- * @param prefix - The prefix that we are looking for
- */
- public void setPrefix(String prefix) {
- this.prefix = prefix;
- }
-
- /**
- * Gets args.
- * @return T
- */
- public T getArgs() {
- return args;
- }
-
- /**
- * Sets args.
- * @param args T
- */
- public void setArgs(T args) {
- this.args = args;
- }
-
- /**
- * Checks if we are doing a rootScan.
- * @return - RootScan.
- */
- public boolean isRootScan() {
- return rootScan;
- }
-
- /**
- * Sets the RootScan property.
- * @param rootScan - Boolean.
- */
- public void setRootScan(boolean rootScan) {
- this.rootScan = rootScan;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
deleted file mode 100644
index 76aa286..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/ServiceFilter.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import com.sun.jersey.spi.container.ContainerRequest;
-import com.sun.jersey.spi.container.ContainerRequestFilter;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.ext.Provider;
-
-/**
- * This class is used to intercept root URL requests and route it to
- * Volume List functionality.
- */
-@Provider
-public class ServiceFilter implements ContainerRequestFilter {
- /**
- * Filter the request.
- * <p>
- * An implementation may modify the state of the request or
- * create a new instance.
- *
- * @param request the request.
- *
- * @return the request.
- */
- @Override
- public ContainerRequest filter(ContainerRequest request) {
- if (request.getRequestUri().getPath().length() > 1) {
- return request;
- }
-
- // Just re-route it to volume handler with some hypothetical volume name.
- // volume name is ignored.
-
- request.setUris(request.getBaseUri(),
- UriBuilder.fromUri(request.getRequestUri())
- .path("/service")
- .queryParam("info", Header.OZONE_LIST_QUERY_SERVICE)
- .build());
-
- return request;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
deleted file mode 100644
index 64fe204..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/StorageHandlerBuilder.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
-
-/**
- * This class is responsible for providing a {@link StorageHandler}
- * implementation to object store web handlers.
- */
-@InterfaceAudience.Private
-public final class StorageHandlerBuilder {
-
- private static final ThreadLocal<StorageHandler>
- STORAGE_HANDLER_THREAD_LOCAL = new ThreadLocal<>();
-
- /**
- * Returns the configured StorageHandler from thread-local storage for this
- * thread.
- *
- * @return StorageHandler from thread-local storage
- */
- public static StorageHandler getStorageHandler() {
- StorageHandler storageHandler = STORAGE_HANDLER_THREAD_LOCAL.get();
- if (storageHandler != null) {
- return storageHandler;
- } else {
- // This only happens while using mvn jetty:run for testing.
- Configuration conf = new OzoneConfiguration();
- return new LocalStorageHandler(conf);
- }
- }
-
- /**
- * Removes the configured StorageHandler from thread-local storage for this
- * thread.
- */
- public static void removeStorageHandler() {
- STORAGE_HANDLER_THREAD_LOCAL.remove();
- }
-
- /**
- * Sets the configured StorageHandler in thread-local storage for this thread.
- *
- * @param storageHandler StorageHandler to set in thread-local storage
- */
- public static void setStorageHandler(StorageHandler storageHandler) {
- STORAGE_HANDLER_THREAD_LOCAL.set(storageHandler);
- }
-
- /**
- * There is no reason to instantiate this class.
- */
- private StorageHandlerBuilder() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
deleted file mode 100644
index 07856d0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserArgs.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * UserArgs is used to package caller info
- * and pass it down to file system.
- */
-@InterfaceAudience.Private
-public class UserArgs {
- private String userName;
- private final String requestID;
- private final String hostName;
- private final UriInfo uri;
- private final Request request;
- private final HttpHeaders headers;
- private String[] groups;
-
-
- /**
- * Constructs user args.
- *
- * @param userName - User name
- * @param requestID - Request ID
- * @param hostName - Host Name
- * @param req - Request
- * @param info - Uri Info
- * @param httpHeaders - http headers
- */
- public UserArgs(String userName, String requestID, String hostName,
- Request req, UriInfo info, HttpHeaders httpHeaders) {
- this.hostName = hostName;
- this.userName = userName;
- this.requestID = requestID;
- this.uri = info;
- this.request = req;
- this.headers = httpHeaders;
- }
-
- /**
- * Constructs user args when we don't know the user name yet.
- *
- * @param requestID _ Request ID
- * @param hostName - Host Name
- * @param req - Request
- * @param info - UriInfo
- * @param httpHeaders - http headers
- */
- public UserArgs(String requestID, String hostName, Request req, UriInfo info,
- HttpHeaders httpHeaders) {
- this.hostName = hostName;
- this.requestID = requestID;
- this.uri = info;
- this.request = req;
- this.headers = httpHeaders;
- }
-
- /**
- * Returns hostname.
- *
- * @return String
- */
- public String getHostName() {
- return hostName;
- }
-
- /**
- * Returns RequestID.
- *
- * @return Long
- */
- public String getRequestID() {
- return requestID;
- }
-
- /**
- * Returns User Name.
- *
- * @return String
- */
- public String getUserName() {
- return userName;
- }
-
- /**
- * Sets the user name.
- *
- * @param userName Name of the user
- */
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- /**
- * Returns list of groups.
- *
- * @return String[]
- */
- public String[] getGroups() {
- return this.groups;
- }
-
- /**
- * Sets the group list.
- *
- * @param groups list of groups
- */
- public void setGroups(String[] groups) {
- this.groups = groups;
- }
-
- /**
- * Returns the resource Name.
- *
- * @return String Resource.
- */
- public String getResourceName() {
- return getUserName();
- }
-
- /**
- * Returns Http Headers for this call.
- *
- * @return httpHeaders
- */
- public HttpHeaders getHeaders() {
- return headers;
- }
-
- /**
- * Returns Request Object.
- *
- * @return Request
- */
- public Request getRequest() {
- return request;
- }
-
- /**
- * Returns UriInfo.
- *
- * @return UriInfo
- */
- public UriInfo getUri() {
- return uri;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
deleted file mode 100644
index d9051f3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/UserHandlerBuilder.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.userauth.Simple;
-
-/**
- * This class is responsible for providing a
- * {@link org.apache.hadoop.ozone.web.interfaces.UserAuth}
- * implementation to object store web handlers.
- */
-@InterfaceAudience.Private
-public final class UserHandlerBuilder {
-
- private static final ThreadLocal<UserAuth> USER_AUTH_THREAD_LOCAL =
- new ThreadLocal<UserAuth>();
-
- /**
- * Returns the configured UserAuth from thread-local storage for this
- * thread.
- *
- * @return UserAuth from thread-local storage
- */
- public static UserAuth getAuthHandler() {
- UserAuth authHandler = USER_AUTH_THREAD_LOCAL.get();
- if (authHandler != null) {
- return authHandler;
- } else {
- // This only happens while using mvn jetty:run for testing.
- return new Simple();
- }
- }
-
- /**
- * Removes the configured UserAuth from thread-local storage for this
- * thread.
- */
- public static void removeAuthHandler() {
- USER_AUTH_THREAD_LOCAL.remove();
- }
-
- /**
- * Sets the configured UserAuthHandler in thread-local storage for this
- * thread.
- *
- * @param authHandler authHandler to set in thread-local storage
- */
- public static void setAuthHandler(UserAuth authHandler) {
- USER_AUTH_THREAD_LOCAL.set(authHandler);
- }
-
- /**
- * There is no reason to instantiate this class.
- */
- private UserHandlerBuilder() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
deleted file mode 100644
index 1d67c67..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeArgs.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * VolumeArgs is used to package all volume
- * related arguments in the call to underlying
- * file system.
- */
-@InterfaceAudience.Private
-public class VolumeArgs extends UserArgs {
- private String adminName;
- private final String volumeName;
- private OzoneQuota quota;
-
- /**
- * Returns Quota Information.
- *
- * @return Quota
- */
- public OzoneQuota getQuota() {
- return quota;
- }
-
- /**
- * Returns volume name.
- *
- * @return String
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Constructs volume Args.
- *
- * @param userName - User name
- * @param volumeName - volume Name
- * @param requestID _ Request ID
- * @param hostName - Host Name
- * @param request - Http Request
- * @param info - URI info
- * @param headers - http headers
- * @param groups - list of groups allowed to access the volume
- */
- public VolumeArgs(String userName, String volumeName, String requestID,
- String hostName, Request request, UriInfo info,
- HttpHeaders headers, String[] groups) {
- super(userName, requestID, hostName, request, info, headers);
- super.setGroups(groups);
- this.volumeName = volumeName;
- }
-
- /**
- * Constructs volume Args.
- *
- * @param volumeName - volume Name
- * @param userArgs - userArgs
- */
- public VolumeArgs(String volumeName, UserArgs userArgs) {
- this(userArgs.getUserName(), volumeName, userArgs.getRequestID(),
- userArgs.getHostName(), userArgs.getRequest(), userArgs.getUri(),
- userArgs.getHeaders(), userArgs.getGroups());
- }
-
- /**
- * Creates VolumeArgs from another VolumeArgs.
- */
- public VolumeArgs(VolumeArgs volArgs) {
- this(volArgs.getVolumeName(), volArgs);
- }
-
- /**
- * Sets Quota information.
- *
- * @param quota - Quota Sting
- * @throws IllegalArgumentException
- */
- public void setQuota(String quota) throws IllegalArgumentException {
- this.quota = OzoneQuota.parseQuota(quota);
- }
-
- /**
- * Sets quota information.
- *
- * @param quota - OzoneQuota
- */
- public void setQuota(OzoneQuota quota) {
- this.quota = quota;
- }
-
- /**
- * Gets admin Name.
- *
- * @return - Admin Name
- */
- public String getAdminName() {
- return adminName;
- }
-
- /**
- * Sets Admin Name.
- *
- * @param adminName - Admin Name
- */
- public void setAdminName(String adminName) {
- this.adminName = adminName;
- }
-
- /**
- * Returns UserName/VolumeName.
- *
- * @return String
- */
- @Override
- public String getResourceName() {
- return super.getResourceName() + "/" + getVolumeName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
deleted file mode 100644
index 7c50859..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeHandler.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.interfaces.Volume;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_FUNCTION;
-
-/**
- * VolumeHandler handles volume specific HTTP calls.
- *
- * Most functions in this file follow a simple pattern.
- * All calls are handled by VolumeProcessTemplate.handleCall, which
- * calls back into doProcess function.
- *
- * Everything common to volume handling is abstracted out in handleCall function
- * For Example : Checking that volume name is sane, we have a supported
- * ozone version number and a valid date. That is everything common is in
- * handleCall and actions specific to a call is inside doProcess callback.
- */
-@InterfaceAudience.Private
-public class VolumeHandler implements Volume {
- private static final Logger LOG = LoggerFactory.getLogger(VolumeHandler
- .class);
- /**
- * Creates a volume.
- *
- * @param volume Volume Name, this has to be unique at Ozone cluster level
- * @param quota Quota for this Storage Volume - <int>(<BYTES|MB|GB|TB>)
- * @param req Request Object
- * @param uriInfo URI info
- * @param headers Http Headers
- *
- * @return Standard JAX-RS Response
- *
- * @throws OzoneException
- */
- @Override
- public Response createVolume(String volume, final String quota, Request req,
- UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "createVolume");
- return new VolumeProcessTemplate() {
- @Override
- public Response doProcess(VolumeArgs args)
- throws IOException, OzoneException {
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- if (auth.isAdmin(args)) {
- args.setAdminName(args.getUserName());
- String volumeOwner = auth.getOzoneUser(args);
-
- if (volumeOwner == null) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
- }
-
- if (!auth.isUser(volumeOwner, args)) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
- }
-
- args.setUserName(volumeOwner);
- args.setGroups(auth.getGroups(args));
- if (!quota.equals(Header.OZONE_QUOTA_UNDEFINED)) {
- setQuotaArgs(args, quota);
- }
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- fs.createVolume(args);
- return OzoneUtils.getResponse(args, HTTP_CREATED, "");
- } else {
- throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
- }
- }
- }.handleCall(volume, req, uriInfo, headers);
- }
-
- /**
- * Updates volume metadata.
- *
- * There are only two actions possible currently with updateVolume.
- * Change the volume ownership or update quota. if you make a call
- * with neither of these actions, update just returns 200 OK.
- *
- * @param volume Volume Name, this has to be unique at Ozone Level
- * @param quota Quota for this volume - <int>(<BYTES|MB|GB|TB>)|remove
- * @param req - Request Object
- * @param uriInfo - URI info
- * @param headers Http Headers
- *
- * @return Standard JAX-RS Response
- *
- * @throws OzoneException
- */
- @Override
- public Response updateVolume(String volume, final String quota, Request req,
- UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
- MDC.put(OZONE_FUNCTION, "updateVolume");
- return new VolumeProcessTemplate() {
- @Override
- public Response doProcess(VolumeArgs args)
- throws IOException, OzoneException {
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- if (auth.isAdmin(args)) {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- args.setAdminName(args.getUserName());
- String newVolumeOwner = auth.getOzoneUser(args);
-
- if (newVolumeOwner != null) {
- if (!auth.isUser(newVolumeOwner, args)) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args);
- }
- args.setUserName(newVolumeOwner);
- fs.setVolumeOwner(args);
- }
-
- if (!quota.equals(Header.OZONE_QUOTA_UNDEFINED)) {
- if (quota.equals(Header.OZONE_QUOTA_REMOVE)) {
- // if it is remove, just tell the file system to remove quota
- fs.setVolumeQuota(args, true);
- } else {
- setQuotaArgs(args, quota);
- fs.setVolumeQuota(args, false);
- }
- }
- return OzoneUtils.getResponse(args, HTTP_OK, "");
- } else {
- // Only Admins are allowed to update volumes
- throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
- }
- }
- }.handleCall(volume, req, uriInfo, headers);
- }
-
-
- /**
- * Deletes a volume if it is empty.
- *
- * @param volume Volume Name
- * @param req - Http Request
- * @param uriInfo - http URI
- * @param headers - http headers
- *
- * @return Standard JAX-RS Response
- *
- * @throws OzoneException
- */
- @Override
- public Response deleteVolume(String volume, Request req, UriInfo uriInfo,
- HttpHeaders headers) throws OzoneException {
- MDC.put(OZONE_FUNCTION, "deleteVolume");
-
- return new VolumeProcessTemplate() {
- @Override
- public Response doProcess(VolumeArgs args)
- throws IOException, OzoneException {
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- if (auth.isAdmin(args)) {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- fs.deleteVolume(args);
- return OzoneUtils.getResponse(args, HTTP_OK, "");
- } else {
- throw ErrorTable.newError(ErrorTable.ACCESS_DENIED, args);
- }
- }
- }.handleCall(volume, req, uriInfo, headers);
- }
-
- /**
- * Returns Volume info. This API can be invoked either by admin or the owner
- *
- * @param volume - Storage Volume Name
- * @param info - Info attribute
- * @param prefix - Prefix key
- * @param maxKeys - Max results
- * @param prevKey - PrevKey
- * @param req - Http Req
- * @param uriInfo - UriInfo.
- * @param headers - Http headers
- * @return
- * @throws OzoneException
- */
- @Override
- public Response getVolumeInfo(String volume, final String info,
- final String prefix,
- final int maxKeys,
- final String prevKey,
- final boolean rootScan,
- Request req,
- final UriInfo uriInfo, HttpHeaders headers)
- throws OzoneException {
-
- return new VolumeProcessTemplate() {
- @Override
- public Response doProcess(VolumeArgs args)
- throws IOException, OzoneException {
-
- switch (info) {
- case Header.OZONE_INFO_QUERY_BUCKET:
- MDC.put(OZONE_FUNCTION, "ListBucket");
- return getBucketsInVolume(args, prefix, maxKeys, prevKey);
- case Header.OZONE_INFO_QUERY_VOLUME:
- MDC.put(OZONE_FUNCTION, "InfoVolume");
- assertNoListParamPresent(uriInfo, args);
- return getVolumeInfoResponse(args); // Return volume info
- case Header.OZONE_LIST_QUERY_SERVICE:
- MDC.put(OZONE_FUNCTION, "ListVolume");
- return getVolumesByUser(args, prefix, maxKeys, prevKey, rootScan);
- default:
- LOG.debug("Unrecognized query param : {} ", info);
- OzoneException ozoneException =
- ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
- ozoneException.setMessage("Unrecognized query param : " + info);
- throw ozoneException;
- }
- }
- }.handleCall(volume, req, uriInfo, headers);
- }
-
- /**
- * Asserts no list query param is present during this call.
- *
- * @param uriInfo - UriInfo. - UriInfo
- * @param args - Volume Args - VolumeArgs.
- * @throws OzoneException
- */
- private void assertNoListParamPresent(final UriInfo uriInfo, VolumeArgs
- args) throws
- OzoneException {
-
- String prefix = uriInfo.getQueryParameters().getFirst("prefix");
- String maxKeys = uriInfo.getQueryParameters().getFirst("max_keys");
- String prevKey = uriInfo.getQueryParameters().getFirst("prev_key");
- if ((prefix != null && !prefix.equals(Header.OZONE_EMPTY_STRING)) ||
- (maxKeys != null && !maxKeys.equals(Header.OZONE_DEFAULT_LIST_SIZE)) ||
- (prevKey != null && !prevKey.equals(Header.OZONE_EMPTY_STRING))) {
- throw ErrorTable.newError(ErrorTable.INVALID_QUERY_PARAM, args);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
deleted file mode 100644
index abeac10..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyArgs.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-
-/**
- * Args for key. Client use this to specify key's attributes on key creation
- * (putKey()).
- */
-public final class KsmKeyArgs {
- private final String volumeName;
- private final String bucketName;
- private final String keyName;
- private long dataSize;
- private final ReplicationType type;
- private final ReplicationFactor factor;
-
- private KsmKeyArgs(String volumeName, String bucketName, String keyName,
- long dataSize, ReplicationType type, ReplicationFactor factor) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.keyName = keyName;
- this.dataSize = dataSize;
- this.type = type;
- this.factor = factor;
- }
-
- public ReplicationType getType() {
- return type;
- }
-
- public ReplicationFactor getFactor() {
- return factor;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public String getBucketName() {
- return bucketName;
- }
-
- public String getKeyName() {
- return keyName;
- }
-
- public long getDataSize() {
- return dataSize;
- }
-
- public void setDataSize(long size) {
- dataSize = size;
- }
-
- /**
- * Builder class of KsmKeyArgs.
- */
- public static class Builder {
- private String volumeName;
- private String bucketName;
- private String keyName;
- private long dataSize;
- private ReplicationType type;
- private ReplicationFactor factor;
-
-
- public Builder setVolumeName(String volume) {
- this.volumeName = volume;
- return this;
- }
-
- public Builder setBucketName(String bucket) {
- this.bucketName = bucket;
- return this;
- }
-
- public Builder setKeyName(String key) {
- this.keyName = key;
- return this;
- }
-
- public Builder setDataSize(long size) {
- this.dataSize = size;
- return this;
- }
-
- public Builder setType(ReplicationType replicationType) {
- this.type = replicationType;
- return this;
- }
-
- public Builder setFactor(ReplicationFactor replicationFactor) {
- this.factor = replicationFactor;
- return this;
- }
-
- public KsmKeyArgs build() {
- return new KsmKeyArgs(volumeName, bucketName, keyName, dataSize,
- type, factor);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
deleted file mode 100644
index 41d523c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyInfo.java
+++ /dev/null
@@ -1,243 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.util.Time;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Args for key block. The block instance for the key requested in putKey.
- * This is returned from KSM to client, and client use class to talk to
- * datanode. Also, this is the metadata written to ksm.db on server side.
- */
-public final class KsmKeyInfo {
- private final String volumeName;
- private final String bucketName;
- // name of key client specified
- private final String keyName;
- private long dataSize;
- private List<KsmKeyLocationInfoGroup> keyLocationVersions;
- private final long creationTime;
- private long modificationTime;
-
- private KsmKeyInfo(String volumeName, String bucketName, String keyName,
- List<KsmKeyLocationInfoGroup> versions, long dataSize,
- long creationTime, long modificationTime) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.keyName = keyName;
- this.dataSize = dataSize;
- // it is important that the versions are ordered from old to new.
- // Do this sanity check when versions got loaded on creating KsmKeyInfo.
- // TODO : this is not necessary, here only because versioning is still a
- // work in-progress, remove this following check when versioning is
- // complete and prove correctly functioning
- long currentVersion = -1;
- for (KsmKeyLocationInfoGroup version : versions) {
- Preconditions.checkArgument(
- currentVersion + 1 == version.getVersion());
- currentVersion = version.getVersion();
- }
- this.keyLocationVersions = versions;
- this.creationTime = creationTime;
- this.modificationTime = modificationTime;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public String getBucketName() {
- return bucketName;
- }
-
- public String getKeyName() {
- return keyName;
- }
-
- public long getDataSize() {
- return dataSize;
- }
-
- public void setDataSize(long size) {
- this.dataSize = size;
- }
-
- public synchronized KsmKeyLocationInfoGroup getLatestVersionLocations()
- throws IOException {
- return keyLocationVersions.size() == 0? null :
- keyLocationVersions.get(keyLocationVersions.size() - 1);
- }
-
- public List<KsmKeyLocationInfoGroup> getKeyLocationVersions() {
- return keyLocationVersions;
- }
-
- public void updateModifcationTime() {
- this.modificationTime = Time.monotonicNow();
- }
-
- /**
- * Append a set of blocks to the latest version. Note that these blocks are
- * part of the latest version, not a new version.
- *
- * @param newLocationList the list of new blocks to be added.
- * @throws IOException
- */
- public synchronized void appendNewBlocks(
- List<KsmKeyLocationInfo> newLocationList) throws IOException {
- if (keyLocationVersions.size() == 0) {
- throw new IOException("Appending new block, but no version exist");
- }
- KsmKeyLocationInfoGroup currentLatestVersion =
- keyLocationVersions.get(keyLocationVersions.size() - 1);
- currentLatestVersion.appendNewBlocks(newLocationList);
- setModificationTime(Time.now());
- }
-
- /**
- * Add a new set of blocks. The new blocks will be added as appending a new
- * version to the all version list.
- *
- * @param newLocationList the list of new blocks to be added.
- * @throws IOException
- */
- public synchronized long addNewVersion(
- List<KsmKeyLocationInfo> newLocationList) throws IOException {
- long latestVersionNum;
- if (keyLocationVersions.size() == 0) {
- // no version exist, these blocks are the very first version.
- keyLocationVersions.add(new KsmKeyLocationInfoGroup(0, newLocationList));
- latestVersionNum = 0;
- } else {
- // it is important that the new version are always at the tail of the list
- KsmKeyLocationInfoGroup currentLatestVersion =
- keyLocationVersions.get(keyLocationVersions.size() - 1);
- // the new version is created based on the current latest version
- KsmKeyLocationInfoGroup newVersion =
- currentLatestVersion.generateNextVersion(newLocationList);
- keyLocationVersions.add(newVersion);
- latestVersionNum = newVersion.getVersion();
- }
- setModificationTime(Time.now());
- return latestVersionNum;
- }
-
- public long getCreationTime() {
- return creationTime;
- }
-
- public long getModificationTime() {
- return modificationTime;
- }
-
- public void setModificationTime(long modificationTime) {
- this.modificationTime = modificationTime;
- }
-
- /**
- * Builder of KsmKeyInfo.
- */
- public static class Builder {
- private String volumeName;
- private String bucketName;
- private String keyName;
- private long dataSize;
- private List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoGroups;
- private long creationTime;
- private long modificationTime;
-
- public Builder setVolumeName(String volume) {
- this.volumeName = volume;
- return this;
- }
-
- public Builder setBucketName(String bucket) {
- this.bucketName = bucket;
- return this;
- }
-
- public Builder setKeyName(String key) {
- this.keyName = key;
- return this;
- }
-
- public Builder setKsmKeyLocationInfos(
- List<KsmKeyLocationInfoGroup> ksmKeyLocationInfoList) {
- this.ksmKeyLocationInfoGroups = ksmKeyLocationInfoList;
- return this;
- }
-
- public Builder setDataSize(long size) {
- this.dataSize = size;
- return this;
- }
-
- public Builder setCreationTime(long crTime) {
- this.creationTime = crTime;
- return this;
- }
-
- public Builder setModificationTime(long mTime) {
- this.modificationTime = mTime;
- return this;
- }
-
- public KsmKeyInfo build() {
- return new KsmKeyInfo(
- volumeName, bucketName, keyName, ksmKeyLocationInfoGroups,
- dataSize, creationTime, modificationTime);
- }
- }
-
- public KeyInfo getProtobuf() {
- long latestVersion = keyLocationVersions.size() == 0 ? -1 :
- keyLocationVersions.get(keyLocationVersions.size() - 1).getVersion();
- return KeyInfo.newBuilder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(dataSize)
- .addAllKeyLocationList(keyLocationVersions.stream()
- .map(KsmKeyLocationInfoGroup::getProtobuf)
- .collect(Collectors.toList()))
- .setLatestVersion(latestVersion)
- .setCreationTime(creationTime)
- .setModificationTime(modificationTime)
- .build();
- }
-
- public static KsmKeyInfo getFromProtobuf(KeyInfo keyInfo) {
- return new KsmKeyInfo(
- keyInfo.getVolumeName(),
- keyInfo.getBucketName(),
- keyInfo.getKeyName(),
- keyInfo.getKeyLocationListList().stream()
- .map(KsmKeyLocationInfoGroup::getFromProtobuf)
- .collect(Collectors.toList()),
- keyInfo.getDataSize(),
- keyInfo.getCreationTime(),
- keyInfo.getModificationTime());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
deleted file mode 100644
index 9d24b30..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfo.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocation;
-
-/**
- * One key can be too huge to fit in one container. In which case it gets split
- * into a number of subkeys. This class represents one such subkey instance.
- */
-public final class KsmKeyLocationInfo {
- private final String containerName;
- // name of the block id SCM assigned for the key
- private final String blockID;
- private final boolean shouldCreateContainer;
- // the id of this subkey in all the subkeys.
- private final long length;
- private final long offset;
- // the version number indicating when this block was added
- private long createVersion;
-
- private KsmKeyLocationInfo(String containerName,
- String blockID, boolean shouldCreateContainer,
- long length, long offset) {
- this.containerName = containerName;
- this.blockID = blockID;
- this.shouldCreateContainer = shouldCreateContainer;
- this.length = length;
- this.offset = offset;
- }
-
- public void setCreateVersion(long version) {
- createVersion = version;
- }
-
- public long getCreateVersion() {
- return createVersion;
- }
-
- public String getContainerName() {
- return containerName;
- }
-
- public String getBlockID() {
- return blockID;
- }
-
- public boolean getShouldCreateContainer() {
- return shouldCreateContainer;
- }
-
- public long getLength() {
- return length;
- }
-
- public long getOffset() {
- return offset;
- }
-
- /**
- * Builder of KsmKeyLocationInfo.
- */
- public static class Builder {
- private String containerName;
- private String blockID;
- private boolean shouldCreateContainer;
- private long length;
- private long offset;
-
- public Builder setContainerName(String container) {
- this.containerName = container;
- return this;
- }
-
- public Builder setBlockID(String block) {
- this.blockID = block;
- return this;
- }
-
- public Builder setShouldCreateContainer(boolean create) {
- this.shouldCreateContainer = create;
- return this;
- }
-
- public Builder setLength(long len) {
- this.length = len;
- return this;
- }
-
- public Builder setOffset(long off) {
- this.offset = off;
- return this;
- }
-
- public KsmKeyLocationInfo build() {
- return new KsmKeyLocationInfo(containerName, blockID,
- shouldCreateContainer, length, offset);
- }
- }
-
- public KeyLocation getProtobuf() {
- return KeyLocation.newBuilder()
- .setContainerName(containerName)
- .setBlockID(blockID)
- .setShouldCreateContainer(shouldCreateContainer)
- .setLength(length)
- .setOffset(offset)
- .setCreateVersion(createVersion)
- .build();
- }
-
- public static KsmKeyLocationInfo getFromProtobuf(KeyLocation keyLocation) {
- KsmKeyLocationInfo info = new KsmKeyLocationInfo(
- keyLocation.getContainerName(),
- keyLocation.getBlockID(),
- keyLocation.getShouldCreateContainer(),
- keyLocation.getLength(),
- keyLocation.getOffset());
- info.setCreateVersion(keyLocation.getCreateVersion());
- return info;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
deleted file mode 100644
index bef65ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmKeyLocationInfoGroup.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyLocationList;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * A list of key locations. This class represents one single version of the
- * blocks of a key.
- */
-public class KsmKeyLocationInfoGroup {
- private final long version;
- private final List<KsmKeyLocationInfo> locationList;
-
- public KsmKeyLocationInfoGroup(long version,
- List<KsmKeyLocationInfo> locations) {
- this.version = version;
- this.locationList = locations;
- }
-
- /**
- * Return only the blocks that are created in the most recent version.
- *
- * @return the list of blocks that are created in the latest version.
- */
- public List<KsmKeyLocationInfo> getBlocksLatestVersionOnly() {
- List<KsmKeyLocationInfo> list = new ArrayList<>();
- locationList.stream().filter(x -> x.getCreateVersion() == version)
- .forEach(list::add);
- return list;
- }
-
- public long getVersion() {
- return version;
- }
-
- public List<KsmKeyLocationInfo> getLocationList() {
- return locationList;
- }
-
- public KeyLocationList getProtobuf() {
- return KeyLocationList.newBuilder()
- .setVersion(version)
- .addAllKeyLocations(
- locationList.stream().map(KsmKeyLocationInfo::getProtobuf)
- .collect(Collectors.toList()))
- .build();
- }
-
- public static KsmKeyLocationInfoGroup getFromProtobuf(
- KeyLocationList keyLocationList) {
- return new KsmKeyLocationInfoGroup(
- keyLocationList.getVersion(),
- keyLocationList.getKeyLocationsList().stream()
- .map(KsmKeyLocationInfo::getFromProtobuf)
- .collect(Collectors.toList()));
- }
-
- /**
- * Given a new block location, generate a new version list based upon this
- * one.
- *
- * @param newLocationList a list of new location to be added.
- * @return
- */
- KsmKeyLocationInfoGroup generateNextVersion(
- List<KsmKeyLocationInfo> newLocationList) throws IOException {
- // TODO : revisit if we can do this method more efficiently
- // one potential inefficiency here is that later version always include
- // older ones. e.g. v1 has B1, then v2, v3...will all have B1 and only add
- // more
- List<KsmKeyLocationInfo> newList = new ArrayList<>();
- newList.addAll(locationList);
- for (KsmKeyLocationInfo newInfo : newLocationList) {
- // all these new blocks will have addVersion of current version + 1
- newInfo.setCreateVersion(version + 1);
- newList.add(newInfo);
- }
- return new KsmKeyLocationInfoGroup(version + 1, newList);
- }
-
- void appendNewBlocks(List<KsmKeyLocationInfo> newLocationList)
- throws IOException {
- for (KsmKeyLocationInfo info : newLocationList) {
- info.setCreateVersion(version);
- locationList.add(info);
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append("version:").append(version).append(" ");
- for (KsmKeyLocationInfo kli : locationList) {
- sb.append(kli.getBlockID()).append(" || ");
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
deleted file mode 100644
index 7d9efad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmOzoneAclMap.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
-
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-/**
- * This helper class keeps a map of all user and their permissions.
- */
-public class KsmOzoneAclMap {
- // per Acl Type user:rights map
- private ArrayList<Map<String, OzoneAclRights>> aclMaps;
-
- KsmOzoneAclMap() {
- aclMaps = new ArrayList<>();
- for (OzoneAclType aclType : OzoneAclType.values()) {
- aclMaps.add(aclType.ordinal(), new HashMap<>());
- }
- }
-
- private Map<String, OzoneAclRights> getMap(OzoneAclType type) {
- return aclMaps.get(type.ordinal());
- }
-
- // For a given acl type and user, get the stored acl
- private OzoneAclRights getAcl(OzoneAclType type, String user) {
- return getMap(type).get(user);
- }
-
- // Add a new acl to the map
- public void addAcl(OzoneAclInfo acl) {
- getMap(acl.getType()).put(acl.getName(), acl.getRights());
- }
-
- // for a given acl, check if the user has access rights
- public boolean hasAccess(OzoneAclInfo acl) {
- OzoneAclRights storedRights = getAcl(acl.getType(), acl.getName());
- if (storedRights != null) {
- switch (acl.getRights()) {
- case READ:
- return (storedRights == OzoneAclRights.READ)
- || (storedRights == OzoneAclRights.READ_WRITE);
- case WRITE:
- return (storedRights == OzoneAclRights.WRITE)
- || (storedRights == OzoneAclRights.READ_WRITE);
- case READ_WRITE:
- return (storedRights == OzoneAclRights.READ_WRITE);
- default:
- return false;
- }
- } else {
- return false;
- }
- }
-
- // Convert this map to OzoneAclInfo Protobuf List
- public List<OzoneAclInfo> ozoneAclGetProtobuf() {
- List<OzoneAclInfo> aclList = new LinkedList<>();
- for (OzoneAclType type: OzoneAclType.values()) {
- for (Map.Entry<String, OzoneAclRights> entry :
- aclMaps.get(type.ordinal()).entrySet()) {
- OzoneAclInfo aclInfo = OzoneAclInfo.newBuilder()
- .setName(entry.getKey())
- .setType(type)
- .setRights(entry.getValue())
- .build();
- aclList.add(aclInfo);
- }
- }
-
- return aclList;
- }
-
- // Create map from list of OzoneAclInfos
- public static KsmOzoneAclMap ozoneAclGetFromProtobuf(
- List<OzoneAclInfo> aclList) {
- KsmOzoneAclMap aclMap = new KsmOzoneAclMap();
- for (OzoneAclInfo acl : aclList) {
- aclMap.addAcl(acl);
- }
- return aclMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
deleted file mode 100644
index 6b1020c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmVolumeArgs.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-
-/**
- * A class that encapsulates the KsmVolumeArgs Args.
- */
-public final class KsmVolumeArgs {
- private final String adminName;
- private final String ownerName;
- private final String volume;
- private final long creationTime;
- private final long quotaInBytes;
- private final Map<String, String> keyValueMap;
- private final KsmOzoneAclMap aclMap;
-
- /**
- * Private constructor, constructed via builder.
- * @param adminName - Administrator's name.
- * @param ownerName - Volume owner's name
- * @param volume - volume name
- * @param quotaInBytes - Volume Quota in bytes.
- * @param keyValueMap - keyValue map.
- * @param aclMap - User to access rights map.
- * @param creationTime - Volume creation time.
- */
- private KsmVolumeArgs(String adminName, String ownerName, String volume,
- long quotaInBytes, Map<String, String> keyValueMap,
- KsmOzoneAclMap aclMap, long creationTime) {
- this.adminName = adminName;
- this.ownerName = ownerName;
- this.volume = volume;
- this.quotaInBytes = quotaInBytes;
- this.keyValueMap = keyValueMap;
- this.aclMap = aclMap;
- this.creationTime = creationTime;
- }
-
- /**
- * Returns the Admin Name.
- * @return String.
- */
- public String getAdminName() {
- return adminName;
- }
-
- /**
- * Returns the owner Name.
- * @return String
- */
- public String getOwnerName() {
- return ownerName;
- }
-
- /**
- * Returns the volume Name.
- * @return String
- */
- public String getVolume() {
- return volume;
- }
-
- /**
- * Returns creation time.
- * @return long
- */
- public long getCreationTime() {
- return creationTime;
- }
-
- /**
- * Returns Quota in Bytes.
- * @return long, Quota in bytes.
- */
- public long getQuotaInBytes() {
- return quotaInBytes;
- }
-
- public Map<String, String> getKeyValueMap() {
- return keyValueMap;
- }
-
- public KsmOzoneAclMap getAclMap() {
- return aclMap;
- }
- /**
- * Returns new builder class that builds a KsmVolumeArgs.
- *
- * @return Builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Builder for KsmVolumeArgs.
- */
- public static class Builder {
- private String adminName;
- private String ownerName;
- private String volume;
- private long creationTime;
- private long quotaInBytes;
- private Map<String, String> keyValueMap;
- private KsmOzoneAclMap aclMap;
-
- /**
- * Constructs a builder.
- */
- Builder() {
- keyValueMap = new HashMap<>();
- aclMap = new KsmOzoneAclMap();
- }
-
- public Builder setAdminName(String admin) {
- this.adminName = admin;
- return this;
- }
-
- public Builder setOwnerName(String owner) {
- this.ownerName = owner;
- return this;
- }
-
- public Builder setVolume(String volumeName) {
- this.volume = volumeName;
- return this;
- }
-
- public Builder setCreationTime(long createdOn) {
- this.creationTime = createdOn;
- return this;
- }
-
- public Builder setQuotaInBytes(long quota) {
- this.quotaInBytes = quota;
- return this;
- }
-
- public Builder addMetadata(String key, String value) {
- keyValueMap.put(key, value); // overwrite if present.
- return this;
- }
-
- public Builder addOzoneAcls(OzoneAclInfo acl) throws IOException {
- aclMap.addAcl(acl);
- return this;
- }
-
- /**
- * Constructs a CreateVolumeArgument.
- * @return CreateVolumeArgs.
- */
- public KsmVolumeArgs build() {
- Preconditions.checkNotNull(adminName);
- Preconditions.checkNotNull(ownerName);
- Preconditions.checkNotNull(volume);
- return new KsmVolumeArgs(adminName, ownerName, volume, quotaInBytes,
- keyValueMap, aclMap, creationTime);
- }
- }
-
- public VolumeInfo getProtobuf() {
- List<KeyValue> metadataList = new LinkedList<>();
- for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
- metadataList.add(KeyValue.newBuilder().setKey(entry.getKey()).
- setValue(entry.getValue()).build());
- }
- List<OzoneAclInfo> aclList = aclMap.ozoneAclGetProtobuf();
-
- return VolumeInfo.newBuilder()
- .setAdminName(adminName)
- .setOwnerName(ownerName)
- .setVolume(volume)
- .setQuotaInBytes(quotaInBytes)
- .addAllMetadata(metadataList)
- .addAllVolumeAcls(aclList)
- .setCreationTime(creationTime)
- .build();
- }
-
- public static KsmVolumeArgs getFromProtobuf(VolumeInfo volInfo) {
- Map<String, String> kvMap = volInfo.getMetadataList().stream()
- .collect(Collectors.toMap(KeyValue::getKey,
- KeyValue::getValue));
- KsmOzoneAclMap aclMap =
- KsmOzoneAclMap.ozoneAclGetFromProtobuf(volInfo.getVolumeAclsList());
-
- return new KsmVolumeArgs(volInfo.getAdminName(), volInfo.getOwnerName(),
- volInfo.getVolume(), volInfo.getQuotaInBytes(), kvMap, aclMap,
- volInfo.getCreationTime());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
deleted file mode 100644
index c19c04b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/OpenKeySession.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-/**
- * This class represents a open key "session". A session here means a key is
- * opened by a specific client, the client sends the handler to server, such
- * that servers can recognize this client, and thus know how to close the key.
- */
-public class OpenKeySession {
- private final int id;
- private final KsmKeyInfo keyInfo;
- // the version of the key when it is being opened in this session.
- // a block that has a create version equals to open version means it will
- // be committed only when this open session is closed.
- private long openVersion;
-
- public OpenKeySession(int id, KsmKeyInfo info, long version) {
- this.id = id;
- this.keyInfo = info;
- this.openVersion = version;
- }
-
- public long getOpenVersion() {
- return this.openVersion;
- }
-
- public KsmKeyInfo getKeyInfo() {
- return keyInfo;
- }
-
- public int getId() {
- return id;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
deleted file mode 100644
index 1c76b8a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/ServiceInfo.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.helpers;
-
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
- .ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * ServiceInfo holds the config details of Ozone services.
- */
-public final class ServiceInfo {
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(ServiceInfo.class);
- private static final ObjectWriter WRITER =
- new ObjectMapper().writerWithDefaultPrettyPrinter();
-
- /**
- * Type of node/service.
- */
- private NodeType nodeType;
- /**
- * Hostname of the node in which the service is running.
- */
- private String hostname;
-
- /**
- * List of ports the service listens to.
- */
- private Map<ServicePort.Type, Integer> ports;
-
- /**
- * Default constructor for JSON deserialization.
- */
- public ServiceInfo() {}
-
- /**
- * Constructs the ServiceInfo for the {@code nodeType}.
- * @param nodeType type of node/service
- * @param hostname hostname of the service
- * @param portList list of ports the service listens to
- */
- private ServiceInfo(
- NodeType nodeType, String hostname, List<ServicePort> portList) {
- Preconditions.checkNotNull(nodeType);
- Preconditions.checkNotNull(hostname);
- this.nodeType = nodeType;
- this.hostname = hostname;
- this.ports = new HashMap<>();
- for (ServicePort port : portList) {
- ports.put(port.getType(), port.getValue());
- }
- }
-
- /**
- * Returns the type of node/service.
- * @return node type
- */
- public NodeType getNodeType() {
- return nodeType;
- }
-
- /**
- * Returns the hostname of the service.
- * @return hostname
- */
- public String getHostname() {
- return hostname;
- }
-
- /**
- * Returns ServicePort.Type to port mappings.
- * @return ports
- */
- public Map<ServicePort.Type, Integer> getPorts() {
- return ports;
- }
-
- /**
- * Returns the port for given type, null if the service doesn't support
- * the type.
- *
- * @param type the type of port.
- * ex: RPC, HTTP, HTTPS, etc..
- */
- @JsonIgnore
- public int getPort(ServicePort.Type type) {
- return ports.get(type);
- }
-
- /**
- * Converts {@link ServiceInfo} to KeySpaceManagerProtocolProtos.ServiceInfo.
- *
- * @return KeySpaceManagerProtocolProtos.ServiceInfo
- */
- @JsonIgnore
- public KeySpaceManagerProtocolProtos.ServiceInfo getProtobuf() {
- KeySpaceManagerProtocolProtos.ServiceInfo.Builder builder =
- KeySpaceManagerProtocolProtos.ServiceInfo.newBuilder();
- builder.setNodeType(nodeType)
- .setHostname(hostname)
- .addAllServicePorts(
- ports.entrySet().stream()
- .map(
- entry ->
- ServicePort.newBuilder()
- .setType(entry.getKey())
- .setValue(entry.getValue()).build())
- .collect(Collectors.toList()));
- return builder.build();
- }
-
- /**
- * Converts KeySpaceManagerProtocolProtos.ServiceInfo to {@link ServiceInfo}.
- *
- * @return {@link ServiceInfo}
- */
- @JsonIgnore
- public static ServiceInfo getFromProtobuf(
- KeySpaceManagerProtocolProtos.ServiceInfo serviceInfo) {
- return new ServiceInfo(serviceInfo.getNodeType(),
- serviceInfo.getHostname(),
- serviceInfo.getServicePortsList());
- }
-
- /**
- * Returns a JSON string of this object.
- *
- * @return String - json string
- * @throws IOException
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * Parse a JSON string into ServiceInfo Object.
- *
- * @param jsonString Json String
- * @return BucketInfo
- * @throws IOException
- */
- public static BucketInfo parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
- /**
- * Creates a new builder to build {@link ServiceInfo}.
- * @return {@link ServiceInfo.Builder}
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Builder used to build/construct {@link ServiceInfo}.
- */
- public static class Builder {
-
- private NodeType node;
- private String host;
- private List<ServicePort> portList = new ArrayList<>();
-
-
- /**
- * Sets the node/service type.
- * @param nodeType type of node
- * @return the builder
- */
- public Builder setNodeType(NodeType nodeType) {
- node = nodeType;
- return this;
- }
-
- /**
- * Sets the hostname of the service.
- * @param hostname service hostname
- * @return the builder
- */
- public Builder setHostname(String hostname) {
- host = hostname;
- return this;
- }
-
- /**
- * Adds the service port to the service port list.
- * @param servicePort RPC port
- * @return the builder
- */
- public Builder addServicePort(ServicePort servicePort) {
- portList.add(servicePort);
- return this;
- }
-
-
- /**
- * Builds and returns {@link ServiceInfo} with the set values.
- * @return {@link ServiceInfo}
- */
- public ServiceInfo build() {
- return new ServiceInfo(node, host, portList);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
deleted file mode 100644
index 1a3d486..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/VolumeArgs.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * A class that encapsulates the createVolume Args.
- */
-public final class VolumeArgs {
- private final String adminName;
- private final String ownerName;
- private final String volume;
- private final long quotaInBytes;
- private final Map<String, String> extendedAttributes;
-
- /**
- * Private constructor, constructed via builder.
- *
- * @param adminName - Administrator name.
- * @param ownerName - Volume owner's name
- * @param volume - volume name
- * @param quotaInBytes - Volume Quota in bytes.
- * @param keyValueMap - keyValue map.
- */
- private VolumeArgs(String adminName, String ownerName, String volume,
- long quotaInBytes, Map<String, String> keyValueMap) {
- this.adminName = adminName;
- this.ownerName = ownerName;
- this.volume = volume;
- this.quotaInBytes = quotaInBytes;
- this.extendedAttributes = keyValueMap;
- }
-
- /**
- * Returns the Admin Name.
- *
- * @return String.
- */
- public String getAdminName() {
- return adminName;
- }
-
- /**
- * Returns the owner Name.
- *
- * @return String
- */
- public String getOwnerName() {
- return ownerName;
- }
-
- /**
- * Returns the volume Name.
- *
- * @return String
- */
- public String getVolume() {
- return volume;
- }
-
- /**
- * Returns Quota in Bytes.
- *
- * @return long, Quota in bytes.
- */
- public long getQuotaInBytes() {
- return quotaInBytes;
- }
-
- public Map<String, String> getExtendedAttributes() {
- return extendedAttributes;
- }
-
- static class Builder {
- private String adminName;
- private String ownerName;
- private String volume;
- private long quotaInBytes;
- private Map<String, String> extendedAttributes;
-
- /**
- * Constructs a builder.
- */
- Builder() {
- extendedAttributes = new HashMap<>();
- }
-
- public void setAdminName(String adminName) {
- this.adminName = adminName;
- }
-
- public void setOwnerName(String ownerName) {
- this.ownerName = ownerName;
- }
-
- public void setVolume(String volume) {
- this.volume = volume;
- }
-
- public void setQuotaInBytes(long quotaInBytes) {
- this.quotaInBytes = quotaInBytes;
- }
-
- public void addMetadata(String key, String value) {
- extendedAttributes.put(key, value); // overwrite if present.
- }
-
- /**
- * Constructs a CreateVolumeArgument.
- *
- * @return CreateVolumeArgs.
- */
- public VolumeArgs build() {
- Preconditions.checkNotNull(adminName);
- Preconditions.checkNotNull(ownerName);
- Preconditions.checkNotNull(volume);
- return new VolumeArgs(adminName, ownerName, volume, quotaInBytes,
- extendedAttributes);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
deleted file mode 100644
index ce627a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
deleted file mode 100644
index 7698ee1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-/**
- This package contains client side protocol library to communicate with KSM.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
deleted file mode 100644
index 5da5a27..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/KeySpaceManagerProtocol.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocol;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Protocol to talk to KSM.
- */
-public interface KeySpaceManagerProtocol {
-
- /**
- * Creates a volume.
- * @param args - Arguments to create Volume.
- * @throws IOException
- */
- void createVolume(KsmVolumeArgs args) throws IOException;
-
- /**
- * Changes the owner of a volume.
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
- */
- void setOwner(String volume, String owner) throws IOException;
-
- /**
- * Changes the Quota on a volume.
- * @param volume - Name of the volume.
- * @param quota - Quota in bytes.
- * @throws IOException
- */
- void setQuota(String volume, long quota) throws IOException;
-
- /**
- * Checks if the specified user can access this volume.
- * @param volume - volume
- * @param userAcl - user acls which needs to be checked for access
- * @return true if the user has required access for the volume,
- * false otherwise
- * @throws IOException
- */
- boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException;
-
- /**
- * Gets the volume information.
- * @param volume - Volume name.
- * @return VolumeArgs or exception is thrown.
- * @throws IOException
- */
- KsmVolumeArgs getVolumeInfo(String volume) throws IOException;
-
- /**
- * Deletes an existing empty volume.
- * @param volume - Name of the volume.
- * @throws IOException
- */
- void deleteVolume(String volume) throws IOException;
-
- /**
- * Lists volume owned by a specific user.
- * @param userName - user name
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix, String
- prevKey, int maxKeys) throws IOException;
-
- /**
- * Lists volume all volumes in the cluster.
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- List<KsmVolumeArgs> listAllVolumes(String prefix, String
- prevKey, int maxKeys) throws IOException;
-
- /**
- * Creates a bucket.
- * @param bucketInfo - BucketInfo to create Bucket.
- * @throws IOException
- */
- void createBucket(KsmBucketInfo bucketInfo) throws IOException;
-
- /**
- * Gets the bucket information.
- * @param volumeName - Volume name.
- * @param bucketName - Bucket name.
- * @return KsmBucketInfo or exception is thrown.
- * @throws IOException
- */
- KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * Sets bucket property from args.
- * @param args - BucketArgs.
- * @throws IOException
- */
- void setBucketProperty(KsmBucketArgs args) throws IOException;
-
- /**
- * Open the given key and return an open key session.
- *
- * @param args the args of the key.
- * @return OpenKeySession instance that client uses to talk to container.
- * @throws IOException
- */
- OpenKeySession openKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Commit a key. This will make the change from the client visible. The client
- * is identified by the clientID.
- *
- * @param args the key to commit
- * @param clientID the client identification
- * @throws IOException
- */
- void commitKey(KsmKeyArgs args, int clientID) throws IOException;
-
- /**
- * Allocate a new block, it is assumed that the client is having an open key
- * session going on. This block will be appended to this open key session.
- *
- * @param args the key to append
- * @param clientID the client identification
- * @return an allocated block
- * @throws IOException
- */
- KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
- throws IOException;
-
- /**
- * Look up for the container of an existing key.
- *
- * @param args the args of the key.
- * @return KsmKeyInfo isntacne that client uses to talk to container.
- * @throws IOException
- */
- KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Deletes an existing key.
- *
- * @param args the args of the key.
- * @throws IOException
- */
- void deleteKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Deletes an existing empty bucket from volume.
- * @param volume - Name of the volume.
- * @param bucket - Name of the bucket.
- * @throws IOException
- */
- void deleteBucket(String volume, String bucket) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link KsmBucketInfo}
- * in the given volume. Argument volumeName is required, others
- * are optional.
- *
- * @param volumeName
- * the name of the volume.
- * @param startBucketName
- * the start bucket name, only the buckets whose name is
- * after this value will be included in the result.
- * @param bucketPrefix
- * bucket name prefix, only the buckets whose name has
- * this prefix will be included in the result.
- * @param maxNumOfBuckets
- * the maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<KsmBucketInfo> listBuckets(String volumeName,
- String startBucketName, String bucketPrefix, int maxNumOfBuckets)
- throws IOException;
-
- /**
- * Returns a list of keys represented by {@link KsmKeyInfo}
- * in the given bucket. Argument volumeName, bucketName is required,
- * others are optional.
- *
- * @param volumeName
- * the name of the volume.
- * @param bucketName
- * the name of the bucket.
- * @param startKeyName
- * the start key name, only the keys whose name is
- * after this value will be included in the result.
- * @param keyPrefix
- * key name prefix, only the keys whose name has
- * this prefix will be included in the result.
- * @param maxKeys
- * the maximum number of keys to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of keys.
- * @throws IOException
- */
- List<KsmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKeyName, String keyPrefix, int maxKeys)
- throws IOException;
-
- /**
- * Returns list of Ozone services with its configuration details.
- *
- * @return list of Ozone services
- * @throws IOException
- */
- List<ServiceInfo> getServiceList() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
deleted file mode 100644
index f77e5fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocol/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.protocol;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
deleted file mode 100644
index cc215cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,744 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocolPB;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.AllocateBlockRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.AllocateBlockResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CommitKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.BucketArgs;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetBucketPropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetBucketPropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.LocateKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.LocateKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetVolumePropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListBucketsResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServiceListRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServiceListResponse;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.stream.Collectors;
-
-/**
- * The client side implementation of KeySpaceManagerProtocol.
- */
-
-@InterfaceAudience.Private
-public final class KeySpaceManagerProtocolClientSideTranslatorPB
- implements KeySpaceManagerProtocol, ProtocolTranslator, Closeable {
-
- /**
- * RpcController is not used and hence is set to null.
- */
- private static final RpcController NULL_RPC_CONTROLLER = null;
-
- private final KeySpaceManagerProtocolPB rpcProxy;
-
- /**
- * Constructor for KeySpaceManger Client.
- * @param rpcProxy
- */
- public KeySpaceManagerProtocolClientSideTranslatorPB(
- KeySpaceManagerProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- /**
- * Closes this stream and releases any system resources associated
- * with it. If the stream is already closed then invoking this
- * method has no effect.
- * <p>
- * <p> As noted in {@link AutoCloseable#close()}, cases where the
- * close may fail require careful attention. It is strongly advised
- * to relinquish the underlying resources and to internally
- * <em>mark</em> the {@code Closeable} as closed, prior to throwing
- * the {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
-
- }
-
- /**
- * Creates a volume.
- *
- * @param args - Arguments to create Volume.
- * @throws IOException
- */
- @Override
- public void createVolume(KsmVolumeArgs args) throws IOException {
- CreateVolumeRequest.Builder req =
- CreateVolumeRequest.newBuilder();
- VolumeInfo volumeInfo = args.getProtobuf();
- req.setVolumeInfo(volumeInfo);
-
- final CreateVolumeResponse resp;
- try {
- resp = rpcProxy.createVolume(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Volume creation failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
- */
- @Override
- public void setOwner(String volume, String owner) throws IOException {
- SetVolumePropertyRequest.Builder req =
- SetVolumePropertyRequest.newBuilder();
- req.setVolumeName(volume).setOwnerName(owner);
- final SetVolumePropertyResponse resp;
- try {
- resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Volume owner change failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * Changes the Quota on a volume.
- *
- * @param volume - Name of the volume.
- * @param quota - Quota in bytes.
- * @throws IOException
- */
- @Override
- public void setQuota(String volume, long quota) throws IOException {
- SetVolumePropertyRequest.Builder req =
- SetVolumePropertyRequest.newBuilder();
- req.setVolumeName(volume).setQuotaInBytes(quota);
- final SetVolumePropertyResponse resp;
- try {
- resp = rpcProxy.setVolumeProperty(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Volume quota change failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * Checks if the specified user can access this volume.
- *
- * @param volume - volume
- * @param userAcl - user acls which needs to be checked for access
- * @return true if the user has required access for the volume,
- * false otherwise
- * @throws IOException
- */
- @Override
- public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl) throws
- IOException {
- CheckVolumeAccessRequest.Builder req =
- CheckVolumeAccessRequest.newBuilder();
- req.setVolumeName(volume).setUserAcl(userAcl);
- final CheckVolumeAccessResponse resp;
- try {
- resp = rpcProxy.checkVolumeAccess(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() == Status.ACCESS_DENIED) {
- return false;
- } else if (resp.getStatus() == Status.OK) {
- return true;
- } else {
- throw new
- IOException("Check Volume Access failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * Gets the volume information.
- *
- * @param volume - Volume name.
- * @return KsmVolumeArgs or exception is thrown.
- * @throws IOException
- */
- @Override
- public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
- InfoVolumeRequest.Builder req = InfoVolumeRequest.newBuilder();
- req.setVolumeName(volume);
- final InfoVolumeResponse resp;
- try {
- resp = rpcProxy.infoVolume(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Info Volume failed, error:" + resp.getStatus());
- }
- return KsmVolumeArgs.getFromProtobuf(resp.getVolumeInfo());
- }
-
- /**
- * Deletes an existing empty volume.
- *
- * @param volume - Name of the volume.
- * @throws IOException
- */
- @Override
- public void deleteVolume(String volume) throws IOException {
- DeleteVolumeRequest.Builder req = DeleteVolumeRequest.newBuilder();
- req.setVolumeName(volume);
- final DeleteVolumeResponse resp;
- try {
- resp = rpcProxy.deleteVolume(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Delete Volume failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * Lists volume owned by a specific user.
- *
- * @param userName - user name
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the
- * prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- @Override
- public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix,
- String prevKey, int maxKeys)
- throws IOException {
- ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.setPrefix(prefix);
- }
- if (!Strings.isNullOrEmpty(prevKey)) {
- builder.setPrevKey(prevKey);
- }
- builder.setMaxKeys(maxKeys);
- builder.setUserName(userName);
- builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_USER);
- return listVolume(builder.build());
- }
-
- /**
- * Lists volume all volumes in the cluster.
- *
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the
- * prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- @Override
- public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey,
- int maxKeys) throws IOException {
- ListVolumeRequest.Builder builder = ListVolumeRequest.newBuilder();
- if (!Strings.isNullOrEmpty(prefix)) {
- builder.setPrefix(prefix);
- }
- if (!Strings.isNullOrEmpty(prevKey)) {
- builder.setPrevKey(prevKey);
- }
- builder.setMaxKeys(maxKeys);
- builder.setScope(ListVolumeRequest.Scope.VOLUMES_BY_CLUSTER);
- return listVolume(builder.build());
- }
-
- private List<KsmVolumeArgs> listVolume(ListVolumeRequest request)
- throws IOException {
- final ListVolumeResponse resp;
- try {
- resp = rpcProxy.listVolumes(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() != Status.OK) {
- throw new IOException("List volume failed, error: "
- + resp.getStatus());
- }
-
- List<KsmVolumeArgs> result = Lists.newArrayList();
- for (VolumeInfo volInfo : resp.getVolumeInfoList()) {
- KsmVolumeArgs volArgs = KsmVolumeArgs.getFromProtobuf(volInfo);
- result.add(volArgs);
- }
-
- return resp.getVolumeInfoList().stream()
- .map(item -> KsmVolumeArgs.getFromProtobuf(item))
- .collect(Collectors.toList());
- }
-
- /**
- * Creates a bucket.
- *
- * @param bucketInfo - BucketInfo to create bucket.
- * @throws IOException
- */
- @Override
- public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
- CreateBucketRequest.Builder req =
- CreateBucketRequest.newBuilder();
- BucketInfo bucketInfoProtobuf = bucketInfo.getProtobuf();
- req.setBucketInfo(bucketInfoProtobuf);
-
- final CreateBucketResponse resp;
- try {
- resp = rpcProxy.createBucket(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Bucket creation failed, error: "
- + resp.getStatus());
- }
- }
-
- /**
- * Gets the bucket information.
- *
- * @param volume - Volume name.
- * @param bucket - Bucket name.
- * @return KsmBucketInfo or exception is thrown.
- * @throws IOException
- */
- @Override
- public KsmBucketInfo getBucketInfo(String volume, String bucket)
- throws IOException {
- InfoBucketRequest.Builder req =
- InfoBucketRequest.newBuilder();
- req.setVolumeName(volume);
- req.setBucketName(bucket);
-
- final InfoBucketResponse resp;
- try {
- resp = rpcProxy.infoBucket(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() == Status.OK) {
- return KsmBucketInfo.getFromProtobuf(resp.getBucketInfo());
- } else {
- throw new IOException("Info Bucket failed, error: "
- + resp.getStatus());
- }
- }
-
- /**
- * Sets bucket property from args.
- * @param args - BucketArgs.
- * @throws IOException
- */
- @Override
- public void setBucketProperty(KsmBucketArgs args)
- throws IOException {
- SetBucketPropertyRequest.Builder req =
- SetBucketPropertyRequest.newBuilder();
- BucketArgs bucketArgs = args.getProtobuf();
- req.setBucketArgs(bucketArgs);
- final SetBucketPropertyResponse resp;
- try {
- resp = rpcProxy.setBucketProperty(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Setting bucket property failed, error: "
- + resp.getStatus());
- }
- }
-
- /**
- * List buckets in a volume.
- *
- * @param volumeName
- * @param startKey
- * @param prefix
- * @param count
- * @return
- * @throws IOException
- */
- @Override
- public List<KsmBucketInfo> listBuckets(String volumeName,
- String startKey, String prefix, int count) throws IOException {
- List<KsmBucketInfo> buckets = new ArrayList<>();
- ListBucketsRequest.Builder reqBuilder = ListBucketsRequest.newBuilder();
- reqBuilder.setVolumeName(volumeName);
- reqBuilder.setCount(count);
- if (startKey != null) {
- reqBuilder.setStartKey(startKey);
- }
- if (prefix != null) {
- reqBuilder.setPrefix(prefix);
- }
- ListBucketsRequest request = reqBuilder.build();
- final ListBucketsResponse resp;
- try {
- resp = rpcProxy.listBuckets(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() == Status.OK) {
- buckets.addAll(
- resp.getBucketInfoList().stream()
- .map(KsmBucketInfo::getFromProtobuf)
- .collect(Collectors.toList()));
- return buckets;
- } else {
- throw new IOException("List Buckets failed, error: "
- + resp.getStatus());
- }
- }
-
- /**
- * Create a new open session of the key, then use the returned meta info to
- * talk to data node to actually write the key.
- * @param args the args for the key to be allocated
- * @return a handler to the key, returned client
- * @throws IOException
- */
- @Override
- public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
- LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
- KeyArgs.Builder keyArgs = KeyArgs.newBuilder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setFactor(args.getFactor())
- .setType(args.getType())
- .setKeyName(args.getKeyName());
- if (args.getDataSize() > 0) {
- keyArgs.setDataSize(args.getDataSize());
- }
- req.setKeyArgs(keyArgs.build());
-
- final LocateKeyResponse resp;
- try {
- resp = rpcProxy.createKey(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Create key failed, error:" + resp.getStatus());
- }
- return new OpenKeySession(resp.getID(),
- KsmKeyInfo.getFromProtobuf(resp.getKeyInfo()), resp.getOpenVersion());
- }
-
- @Override
- public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
- throws IOException {
- AllocateBlockRequest.Builder req = AllocateBlockRequest.newBuilder();
- KeyArgs keyArgs = KeyArgs.newBuilder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setFactor(args.getFactor())
- .setType(args.getType())
- .setDataSize(args.getDataSize()).build();
- req.setKeyArgs(keyArgs);
- req.setClientID(clientID);
-
- final AllocateBlockResponse resp;
- try {
- resp = rpcProxy.allocateBlock(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Allocate block failed, error:" +
- resp.getStatus());
- }
- return KsmKeyLocationInfo.getFromProtobuf(resp.getKeyLocation());
- }
-
- @Override
- public void commitKey(KsmKeyArgs args, int clientID)
- throws IOException {
- CommitKeyRequest.Builder req = CommitKeyRequest.newBuilder();
- KeyArgs keyArgs = KeyArgs.newBuilder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setDataSize(args.getDataSize()).build();
- req.setKeyArgs(keyArgs);
- req.setClientID(clientID);
-
- final CommitKeyResponse resp;
- try {
- resp = rpcProxy.commitKey(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Commit key failed, error:" +
- resp.getStatus());
- }
- }
-
-
- @Override
- public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
- LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
- KeyArgs keyArgs = KeyArgs.newBuilder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setDataSize(args.getDataSize()).build();
- req.setKeyArgs(keyArgs);
-
- final LocateKeyResponse resp;
- try {
- resp = rpcProxy.lookupKey(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Lookup key failed, error:" +
- resp.getStatus());
- }
- return KsmKeyInfo.getFromProtobuf(resp.getKeyInfo());
- }
-
- /**
- * Deletes an existing key.
- *
- * @param args the args of the key.
- * @throws IOException
- */
- @Override
- public void deleteKey(KsmKeyArgs args) throws IOException {
- LocateKeyRequest.Builder req = LocateKeyRequest.newBuilder();
- KeyArgs keyArgs = KeyArgs.newBuilder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName()).build();
- req.setKeyArgs(keyArgs);
-
- final LocateKeyResponse resp;
- try {
- resp = rpcProxy.deleteKey(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new IOException("Delete key failed, error:" +
- resp.getStatus());
- }
- }
-
- /**
- * Deletes an existing empty bucket from volume.
- * @param volume - Name of the volume.
- * @param bucket - Name of the bucket.
- * @throws IOException
- */
- public void deleteBucket(String volume, String bucket) throws IOException {
- DeleteBucketRequest.Builder req = DeleteBucketRequest.newBuilder();
- req.setVolumeName(volume);
- req.setBucketName(bucket);
- final DeleteBucketResponse resp;
- try {
- resp = rpcProxy.deleteBucket(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (resp.getStatus() != Status.OK) {
- throw new
- IOException("Delete Bucket failed, error:" + resp.getStatus());
- }
- }
-
- /**
- * List keys in a bucket.
- */
- @Override
- public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String prefix, int maxKeys) throws IOException {
- List<KsmKeyInfo> keys = new ArrayList<>();
- ListKeysRequest.Builder reqBuilder = ListKeysRequest.newBuilder();
- reqBuilder.setVolumeName(volumeName);
- reqBuilder.setBucketName(bucketName);
- reqBuilder.setCount(maxKeys);
-
- if (startKey != null) {
- reqBuilder.setStartKey(startKey);
- }
-
- if (prefix != null) {
- reqBuilder.setPrefix(prefix);
- }
-
- ListKeysRequest request = reqBuilder.build();
- final ListKeysResponse resp;
- try {
- resp = rpcProxy.listKeys(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() == Status.OK) {
- keys.addAll(
- resp.getKeyInfoList().stream()
- .map(KsmKeyInfo::getFromProtobuf)
- .collect(Collectors.toList()));
- return keys;
- } else {
- throw new IOException("List Keys failed, error: "
- + resp.getStatus());
- }
- }
-
- @Override
- public List<ServiceInfo> getServiceList() throws IOException {
- ServiceListRequest request = ServiceListRequest.newBuilder().build();
- final ServiceListResponse resp;
- try {
- resp = rpcProxy.getServiceList(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- if (resp.getStatus() == Status.OK) {
- return resp.getServiceInfoList().stream()
- .map(ServiceInfo::getFromProtobuf)
- .collect(Collectors.toList());
- } else {
- throw new IOException("Getting service list failed, error: "
- + resp.getStatus());
- }
- }
-
- /**
- * Return the proxy object underlying this protocol translator.
- *
- * @return the proxy object underlying this protocol translator.
- */
- @Override
- public Object getUnderlyingProxyObject() {
- return null;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
deleted file mode 100644
index 7306b81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/StorageContainerDatanodeProtocol.proto
+++ /dev/null
@@ -1,367 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-
-option java_outer_classname = "StorageContainerDatanodeProtocolProtos";
-
-option java_generic_services = true;
-
-option java_generate_equals_and_hash = true;
-
-package hadoop.hdfs;
-
-import "hdfs.proto";
-
-import "HdfsServer.proto";
-
-import "DatanodeProtocol.proto";
-
-import "Ozone.proto";
-
-
-/**
-* This message is send by data node to indicate that it is alive or it is
-* registering with the node manager.
-*/
-message SCMHeartbeatRequestProto {
- required DatanodeIDProto datanodeID = 1;
- optional SCMNodeReport nodeReport = 2;
- optional ReportState containerReportState = 3;
-}
-
-enum DatanodeContainerState {
- closed = 0;
- open = 1;
-}
-
-/**
-NodeState contains messages from datanode to SCM saying that it has
-some information that SCM might be interested in.*/
-message ReportState {
- enum states {
- noContainerReports = 0;
- completeContinerReport = 1;
- deltaContainerReport = 2;
- }
- required states state = 1;
- required int64 count = 2 [default = 0];
-}
-
-
-/**
-This message is used to persist the information about a container in the
-SCM database, This information allows SCM to startup faster and avoid having
-all container info in memory all the time.
- */
-message ContainerPersistanceProto {
- required DatanodeContainerState state = 1;
- required hadoop.hdfs.ozone.Pipeline pipeline = 2;
- required ContainerInfo info = 3;
-}
-
-/**
-This message is used to do a quick look up of which containers are effected
-if a node goes down
-*/
-message NodeContianerMapping {
- repeated string contianerName = 1;
-}
-
-/**
-A container report contains the following information.
-*/
-message ContainerInfo {
- required string containerName = 1;
- optional string finalhash = 2;
- optional int64 size = 3;
- optional int64 used = 4;
- optional int64 keyCount = 5;
- // TODO: move the io count to separate message
- optional int64 readCount = 6;
- optional int64 writeCount = 7;
- optional int64 readBytes = 8;
- optional int64 writeBytes = 9;
- required int64 containerID = 10;
- optional hadoop.hdfs.ozone.LifeCycleState state = 11;
-}
-
-// The deleted blocks which are stored in deletedBlock.db of scm.
-message DeletedBlocksTransaction {
- required int64 txID = 1;
- required string containerName = 2;
- repeated string blockID = 3;
- // the retry time of sending deleting command to datanode.
- required int32 count = 4;
-}
-
-/**
-A set of container reports, max count is generally set to
-8192 since that keeps the size of the reports under 1 MB.
-*/
-message ContainerReportsRequestProto {
- enum reportType {
- fullReport = 0;
- deltaReport = 1;
- }
- required DatanodeIDProto datanodeID = 1;
- repeated ContainerInfo reports = 2;
- required reportType type = 3;
-}
-
-message ContainerReportsResponseProto {
-}
-
-/**
-* This message is send along with the heart beat to report datanode
-* storage utilization by SCM.
-*/
-message SCMNodeReport {
- repeated SCMStorageReport storageReport = 1;
-}
-
-message SCMStorageReport {
- required string storageUuid = 1;
- optional uint64 capacity = 2 [default = 0];
- optional uint64 scmUsed = 3 [default = 0];
- optional uint64 remaining = 4 [default = 0];
- optional StorageTypeProto storageType = 5 [default = DISK];
-}
-
-message SCMRegisterRequestProto {
- required DatanodeIDProto datanodeID = 1;
- optional SCMNodeAddressList addressList = 2;
-}
-
-/**
- * Request for version info of the software stack on the server.
- */
-message SCMVersionRequestProto {
-
-}
-
-/**
-* Generic response that is send to a version request. This allows keys to be
-* added on the fly and protocol to remain stable.
-*/
-message SCMVersionResponseProto {
- required uint32 softwareVersion = 1;
- repeated hadoop.hdfs.ozone.KeyValue keys = 2;
-}
-
-message SCMNodeAddressList {
- repeated string addressList = 1;
-}
-
-/**
- * Datanode ID returned by the SCM. This is similar to name node
- * registeration of a datanode.
- */
-message SCMRegisteredCmdResponseProto {
- enum ErrorCode {
- success = 1;
- errorNodeNotPermitted = 2;
- }
- required ErrorCode errorCode = 2;
- optional string datanodeUUID = 3;
- optional string clusterID = 4;
- optional SCMNodeAddressList addressList = 5;
-}
-
-/**
- * SCM informs a datanode to register itself again.
- * With recieving this command, datanode will transit to REGISTER state.
- */
-message SCMReregisterCmdResponseProto {}
-
-/**
- * Container ID maintains the container's Identity along with cluster ID
- * after the registration is done.
- */
-message ContainerNodeIDProto {
- required DatanodeIDProto datanodeID = 1;
- optional string clusterID = 2;
-}
-
-
-
-/**
-This command tells the data node to send in the container report when possible
-*/
-message SendContainerReportProto {
-}
-
-/**
-This command asks the datanode to close a specific container.
-*/
-message SCMCloseContainerCmdResponseProto {
- required string containerName = 1;
-}
-
-/**
-Type of commands supported by SCM to datanode protocol.
-*/
-enum Type {
- versionCommand = 2;
- registeredCommand = 3;
- sendContainerReport = 4;
- reregisterCommand = 5;
- deleteBlocksCommand = 6;
- closeContainerCommand = 7;
-}
-
-/*
- * These are commands returned by SCM for to the datanode to execute.
- */
-message SCMCommandResponseProto {
- required Type cmdType = 2; // Type of the command
- optional SCMRegisteredCmdResponseProto registeredProto = 3;
- optional SCMVersionResponseProto versionProto = 4;
- optional SendContainerReportProto sendReport = 5;
- optional SCMReregisterCmdResponseProto reregisterProto = 6;
- optional SCMDeleteBlocksCmdResponseProto deleteBlocksProto = 7;
- required string datanodeUUID = 8;
- optional SCMCloseContainerCmdResponseProto closeContainerProto = 9;
-}
-
-
-/*
- * A group of commands for the datanode to execute
- */
-message SCMHeartbeatResponseProto {
- repeated SCMCommandResponseProto commands = 1;
-}
-
-// HB response from SCM, contains a list of block deletion transactions.
-message SCMDeleteBlocksCmdResponseProto {
- repeated DeletedBlocksTransaction deletedBlocksTransactions = 1;
-}
-
-// SendACK response returned by datanode to SCM, currently empty.
-message ContainerBlocksDeletionACKResponseProto {
-}
-
-// ACK message datanode sent to SCM, contains the result of
-// block deletion transactions.
-message ContainerBlocksDeletionACKProto {
- message DeleteBlockTransactionResult {
- required int64 txID = 1;
- required bool success = 2;
- }
- repeated DeleteBlockTransactionResult results = 1;
-}
-
-/**
- * Protocol used from a datanode to StorageContainerManager.
- *
- * Please see the request and response messages for details of the RPC calls.
- *
- * Here is a simple state diagram that shows how a datanode would boot up and
- * communicate with SCM.
- *
- * -----------------------
- * | Start |
- * ---------- ------------
- * |
- * |
- * |
- * |
- * |
- * |
- * |
- * ----------v-------------
- * | Searching for SCM ------------
- * ---------- ------------- |
- * | |
- * | |
- * | ----------v-------------
- * | | Register if needed |
- * | ----------- ------------
- * | |
- * v |
- * ----------- ---------------- |
- * --------- Heartbeat state <--------
- * | --------^-------------------
- * | |
- * | |
- * | |
- * | |
- * | |
- * | |
- * | |
- * ------------------
- *
- *
- *
- * Here is how this protocol is used by the datanode. When a datanode boots up
- * it moves into a stated called SEARCHING_SCM. In this state datanode is
- * trying to establish communication with the SCM. The address of the SCMs are
- * retrieved from the configuration information.
- *
- * In the SEARCHING_SCM state, only rpc call made by datanode is a getVersion
- * call to SCM. Once any of the SCMs reply, datanode checks if it has a local
- * persisted datanode ID. If it has this means that this datanode is already
- * registered with some SCM. If this file is not found, datanode assumes that
- * it needs to do a registration.
- *
- * If registration is need datanode moves into REGISTER state. It will
- * send a register call with datanodeID data structure and presist that info.
- *
- * The response to the command contains clusterID. This information is
- * also persisted by the datanode and moves into heartbeat state.
- *
- * Once in the heartbeat state, datanode sends heartbeats and container reports
- * to SCM and process commands issued by SCM until it is shutdown.
- *
- */
-service StorageContainerDatanodeProtocolService {
-
- /**
- * Gets the version information from the SCM.
- */
- rpc getVersion (SCMVersionRequestProto) returns (SCMVersionResponseProto);
-
- /**
- * Registers a data node with SCM.
- */
- rpc register (SCMRegisterRequestProto) returns (SCMRegisteredCmdResponseProto);
-
- /**
- * Send heartbeat from datanode to SCM. HB's under SCM looks more
- * like life line protocol than HB's under HDFS. In other words, it is
- * extremely light weight and contains no data payload.
- */
- rpc sendHeartbeat (SCMHeartbeatRequestProto) returns (SCMHeartbeatResponseProto);
-
- /**
- send container reports sends the container report to SCM. This will
- return a null command as response.
- */
- rpc sendContainerReport(ContainerReportsRequestProto) returns (ContainerReportsResponseProto);
-
- /**
- * Sends the block deletion ACK to SCM.
- */
- rpc sendContainerBlocksDeletionACK (ContainerBlocksDeletionACKProto) returns (ContainerBlocksDeletionACKResponseProto);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
deleted file mode 100644
index 2e103fe..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/META-INF/services/com.sun.jersey.spi.container.ContainerProvider
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainerProvider
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
deleted file mode 100644
index 97eaeae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/ozone-default.xml
+++ /dev/null
@@ -1,1308 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<!-- Do not modify this file directly. Instead, copy entries that you -->
-<!-- wish to modify from this file into ozone-site.xml and change them -->
-<!-- there. If ozone-site.xml does not already exist, create it. -->
-
-<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
-<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
-<!--REST, STORAGE, PIPELINE, STANDALONE -->
-
-<configuration>
- <!--CBlock Settings-->
- <property>
- <name>dfs.cblock.block.buffer.flush.interval</name>
- <value>60s</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Controls the frequency at this the local cache flushes the
- blocks to the remote containers.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.block.buffer.size</name>
- <value>512</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Size of the local cache for blocks. So cache size will be block
- size multiplied by this number.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.core.min.pool.size</name>
- <value>16</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- A minimum number of threads in the pool that cBlock cache will
- use for the background I/O to remote containers.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.max.pool.size</name>
- <value>256</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Maximum number of threads in the pool that cBlock cache will
- use for background I/O to remote containers.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.keep.alive</name>
- <value>60s</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- If the cblock cache has no I/O, then the threads in the cache
- pool are kept idle for this amount of time before shutting down.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.leveldb.cache.size.mb</name>
- <value>256</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- The amount of physical memory allocated to the local cache. The
- SCSI driver will allocate this much RAM cache instances.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.max.retry</name>
- <value>65536</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- If the local cache is enabled then, CBlock writes to the local
- cache when I/O happens. Then the background I/O threads write this
- block to the remote containers. This value controls how many times the
- background thread should attempt to do I/O to the remote containers
- before giving up.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.queue.size.in.kb</name>
- <value>256</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Size of the in memory cache queue, that is flushed to local
- disk.
- </description>
- </property>
- <property>
- <name>dfs.cblock.cache.thread.priority</name>
- <value>5</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Priority of cache flusher thread, affecting the relative performance of
- write and read. Supported values are 1, 5, 10.
- Use 10 for high priority and 1 for low priority.
- </description>
- </property>
- <property>
- <name>dfs.cblock.container.size.gb</name>
- <value>5</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The size of ozone container in the number of GBs. Note that
- this is not setting container size for ozone. This setting is
- instructing CBlock to manage containers at a standard size.
- </description>
- </property>
- <property>
- <name>dfs.cblock.disk.cache.path</name>
- <value>${hadoop.tmp.dir}/cblockCacheDB</value>
- <tag>CBLOCK, REQUIRED</tag>
- <description>
- The default path for the cblock local cache. If the cblock
- local cache is enabled, then it must be set to a valid path. This cache
- *should* be mapped to the fastest disk on a given machine, For example,
- an SSD drive would be a good idea. Currently, all mounted disk on a
- data node is mapped to a single path, so having a large number of IOPS
- is essential.
- </description>
- </property>
- <property>
- <name>dfs.cblock.jscsi-address</name>
- <value/>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The address that cblock will be bind to, should be a host:port
- format, This setting is required for cblock server to start.
- This address to be used by jscsi to mount volume.
- </description>
- </property>
- <property>
- <name>dfs.cblock.jscsi.cblock.server.address</name>
- <value>127.0.0.1</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The address local jscsi server will use to talk to cblock manager.
- </description>
- </property>
- <property>
- <name>dfs.cblock.jscsi.port</name>
- <value>9811</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The port on CBlockManager node for jSCSI to talk to.
- </description>
- </property>
- <property>
- <name>dfs.cblock.jscsi.rpc-bind-host</name>
- <value>0.0.0.0</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The actual address the cblock jscsi rpc server will bind to. If
- this optional address is set, it overrides only the hostname portion of
- dfs.cblock.jscsi-address.
- </description>
- </property>
- <property>
- <name>dfs.cblock.jscsi.server.address</name>
- <value>0.0.0.0</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The address that jscsi server will be running, it is nice have one
- local jscsi server for each client(Linux JSCSI client) that tries to
- mount cblock.
- </description>
- </property>
- <property>
- <name>dfs.cblock.manager.pool.size</name>
- <value>16</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Number of active threads that cblock manager will use for container
- operations. The maximum number of the threads are limited to the
- processor count * 2.
- </description>
- </property>
- <property>
- <name>dfs.cblock.rpc.timeout</name>
- <value>300s</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- RPC timeout used for cblock CLI operations. When you
- create very large disks, like 5TB, etc. The number of containers
- allocated in the system is huge. It is will 5TB/5GB, which is 1000
- containers. The client CLI might timeout even though the cblock manager
- creates the specified disk. This value allows the user to wait for a
- longer period.
- </description>
- </property>
- <property>
- <name>dfs.cblock.scm.ipaddress</name>
- <value>127.0.0.1</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- IP address used by cblock to connect to SCM.
- </description>
- </property>
- <property>
- <name>dfs.cblock.scm.port</name>
- <value>9860</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- Port used by cblock to connect to SCM.
- </description>
- </property>
- <property>
- <name>dfs.cblock.service.handler.count</name>
- <value>10</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- Default number of handlers for CBlock service rpc.
- </description>
- </property>
- <property>
- <name>dfs.cblock.service.leveldb.path</name>
- <value>${hadoop.tmp.dir}/cblock_server.dat</value>
- <tag>CBLOCK, REQUIRED</tag>
- <description>
- Default path for the cblock meta data store.
- </description>
- </property>
- <property>
- <name>dfs.cblock.service.rpc-bind-host</name>
- <value>0.0.0.0</value>
- <tag>CBLOCK, MANAGEMENT</tag>
- <description>
- The actual address the cblock service RPC server will bind to.
- If the optional address is set, it overrides only the hostname portion of
- dfs.cblock.servicerpc-address.
- </description>
- </property>
- <property>
- <name>dfs.cblock.servicerpc-address</name>
- <value/>
- <tag>CBLOCK, MANAGEMENT, REQUIRED</tag>
- <description>
- The address that cblock will be bind to, should be a host:port
- format, this setting is required for cblock server to start.
- This address is used for cblock management operations like create, delete,
- info and list volumes
- </description>
- </property>
- <property>
- <name>dfs.cblock.short.circuit.io</name>
- <value>false</value>
- <tag>CBLOCK, PERFORMANCE</tag>
- <description>
- Enables use of the local cache in cblock. Enabling this allows
- I/O against the local cache and background threads do actual I/O against
- the
- containers.
- </description>
- </property>
- <property>
- <name>dfs.cblock.trace.io</name>
- <value>false</value>
- <tag>CBLOCK, DEBUG</tag>
- <description>Default flag for enabling trace io, Trace I/O logs all I/O with
- hashes of
- data. This is useful for detecting things like data corruption.
- </description>
- </property>
-
- <property>
- <name>dfs.cblock.iscsi.advertised.ip</name>
- <value>0.0.0.0</value>
- <tag>CBLOCK</tag>
- <description>
- IP address returned during the iscsi discovery.
- </description>
- </property>
-
- <property>
- <name>dfs.cblock.iscsi.advertised.port</name>
- <value>3260</value>
- <tag>CBLOCK</tag>
- <description>
- TCP port returned during the iscsi discovery.
- </description>
- </property>
-
- <property>
- <name>dfs.cblock.kubernetes.dynamic-provisioner.enabled</name>
- <value>false</value>
- <tag>CBLOCK, KUBERNETES</tag>
- <description>Flag to enable automatic creation of cblocks and
- kubernetes PersitentVolumes in kubernetes environment.</description>
- </property>
-
- <property>
- <name>dfs.cblock.kubernetes.cblock-user</name>
- <value>iqn.2001-04.org.apache.hadoop</value>
- <tag>CBLOCK, KUBERNETES</tag>
- <description>CBlock user to use for the dynamic provisioner.
- This user will own all of the auto-created cblocks.</description>
- </property>
-
- <property>
- <name>dfs.cblock.kubernetes.configfile</name>
- <value></value>
- <tag>CBLOCK, KUBERNETES</tag>
- <description>Location of the kubernetes configuration file
- to access the kubernetes cluster. Not required inside a pod
- as the default service account will be if this value is
- empty.</description>
- </property>
-
- <property>
- <name>dfs.cblock.iscsi.advertised.ip</name>
- <value></value>
- <tag>CBLOCK, KUBERNETES</tag>
- <description>IP where the cblock target server is available
- from the kubernetes nodes. Usually it's a cluster ip address
- which is defined by a deployed Service.</description>
- </property>
-
- <property>
- <name>dfs.cblock.iscsi.advertised.port</name>
- <value>3260</value>
- <tag>CBLOCK, KUBERNETES</tag>
- <description>Port where the cblock target server is available
- from the kubernetes nodes. Could be different from the
- listening port if jscsi is behind a Service.</description>
- </property>
-
- <!--Container Settings used by Datanode-->
- <property>
- <name>ozone.container.cache.size</name>
- <value>1024</value>
- <tag>PERFORMANCE, CONTAINER, STORAGE</tag>
- <description>The open container is cached on the data node side. We maintain
- an LRU
- cache for caching the recently used containers. This setting controls the
- size of that cache.
- </description>
- </property>
- <property>
- <name>dfs.container.ipc</name>
- <value>9859</value>
- <tag>OZONE, CONTAINER, MANAGEMENT</tag>
- <description>The ipc port number of container.</description>
- </property>
- <property>
- <name>dfs.container.ipc.random.port</name>
- <value>false</value>
- <tag>OZONE, DEBUG, CONTAINER</tag>
- <description>Allocates a random free port for ozone container. This is used
- only while
- running unit tests.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.datanode.storage.dir</name>
- <value/>
- <tag>OZONE, CONTAINER, STORAGE, MANAGEMENT, RATIS</tag>
- <description>This directory is used for storing Ratis metadata like logs. If
- this is
- not set then default metadata dirs is used. A warning will be logged if
- this not set. Ideally, this should be mapped to a fast disk like an SSD.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.enabled</name>
- <value>false</value>
- <tag>OZONE, MANAGEMENT, PIPELINE, RATIS</tag>
- <description>Ozone supports different kinds of replication pipelines. Ratis
- is one of
- the replication pipeline supported by ozone.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.ipc</name>
- <value>9858</value>
- <tag>OZONE, CONTAINER, PIPELINE, RATIS, MANAGEMENT</tag>
- <description>The ipc port number of container.</description>
- </property>
- <property>
- <name>dfs.container.ratis.ipc.random.port</name>
- <value>false</value>
- <tag>OZONE,DEBUG</tag>
- <description>Allocates a random free port for ozone ratis port for the
- container. This
- is used only while running unit tests.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.rpc.type</name>
- <value>GRPC</value>
- <tag>OZONE, RATIS, MANAGEMENT</tag>
- <description>Ratis supports different kinds of transports like netty, GRPC,
- Hadoop RPC
- etc. This picks one of those for this cluster.
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.num.write.chunk.threads</name>
- <value>60</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>Maximum number of threads in the thread pool that Ratis
- will use for writing chunks (60 by default).
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.segment.size</name>
- <value>1073741824</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the raft segment used by Apache Ratis on datanodes.
- (1 GB by default)
- </description>
- </property>
- <property>
- <name>dfs.container.ratis.segment.preallocated.size</name>
- <value>134217728</value>
- <tag>OZONE, RATIS, PERFORMANCE</tag>
- <description>The size of the buffer which is preallocated for raft segment
- used by Apache Ratis on datanodes.(128 MB by default)
- </description>
- </property>
- <property>
- <name>ozone.container.report.interval</name>
- <value>60000ms</value>
- <tag>OZONE, CONTAINER, MANAGEMENT</tag>
- <description>Time interval of the datanode to send container report. Each
- datanode periodically send container report upon receive
- sendContainerReport from SCM. Unit could be defined with
- postfix (ns,ms,s,m,h,d)</description>
- </property>
- <!--Ozone Settings-->
- <property>
- <name>ozone.administrators</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>Ozone administrator users delimited by the comma.
- If not set, only the user who launches an ozone service will be the admin
- user. This property must be set if ozone services are started by different
- users. Otherwise, the RPC layer will reject calls from other servers which
- are started by users not in the list.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.container.limit.per.interval</name>
- <value>10</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A maximum number of containers to be scanned by block deleting
- service per
- time interval. The block deleting service spawns a thread to handle block
- deletions in a container. This property is used to throttle the number of
- threads spawned for block deletions.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.limit.per.task</name>
- <value>1000</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A maximum number of blocks to be deleted by block deleting
- service per
- time interval. This property is used to throttle the actual number of
- block deletions on a data node per container.
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.service.interval</name>
- <value>1m</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>Time interval of the block deleting service.
- The block deleting service runs on each datanode periodically and
- deletes blocks queued for deletion. Unit could be defined with
- postfix (ns,ms,s,m,h,d)
- </description>
- </property>
- <property>
- <name>ozone.block.deleting.service.timeout</name>
- <value>300000ms</value>
- <tag>OZONE, PERFORMANCE, SCM</tag>
- <description>A timeout value of block deletion service. If this is set
- greater than 0,
- the service will stop waiting for the block deleting completion after this
- time. If timeout happens to a large proportion of block deletion, this
- needs to be increased with ozone.block.deleting.limit.per.task. This
- setting supports multiple time unit suffixes as described in
- dfs.heartbeat.interval. If no suffix is specified, then milliseconds is
- assumed.
- </description>
- </property>
- <property>
- <name>ozone.client.connection.timeout</name>
- <value>5000ms</value>
- <tag>OZONE, PERFORMANCE, CLIENT</tag>
- <description>Connection timeout for Ozone client in milliseconds.
- </description>
- </property>
- <property>
- <name>ozone.client.protocol</name>
- <value>org.apache.hadoop.ozone.client.rpc.RpcClient</value>
- <tag>OZONE, CLIENT, MANAGEMENT</tag>
- <description>Protocol class to be used by the client to connect to ozone
- cluster.
- The build-in implementation includes:
- org.apache.hadoop.ozone.client.rpc.RpcClient for RPC
- org.apache.hadoop.ozone.client.rest.RestClient for REST
- The default is the RpClient. Please do not change this unless you have a
- very good understanding of what you are doing.
- </description>
- </property>
- <property>
- <name>ozone.client.socket.timeout</name>
- <value>5000ms</value>
- <tag>OZONE, CLIENT</tag>
- <description>Socket timeout for Ozone client. Unit could be defined with
- postfix (ns,ms,s,m,h,d)</description>
- </property>
- <property>
- <name>ozone.enabled</name>
- <value>false</value>
- <tag>OZONE, REQUIRED</tag>
- <description>
- Status of the Ozone Object Storage service is enabled.
- Set to true to enable Ozone.
- Set to false to disable Ozone.
- Unless this value is set to true, Ozone services will not be started in
- the cluster.
-
- Please note: By default ozone is disabled on a hadoop cluster.
- </description>
- </property>
- <property>
- <name>ozone.handler.type</name>
- <value>distributed</value>
- <tag>OZONE, REST</tag>
- <description>
- Tells ozone which storage handler to use. The possible values are:
- distributed - The Ozone distributed storage handler, which speaks to
- KSM/SCM on the backend and provides REST services to clients.
- local - Local Storage handler strictly for testing - To be removed.
- </description>
- </property>
- <property>
- <name>ozone.key.deleting.limit.per.task</name>
- <value>1000</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- A maximum number of keys to be scanned by key deleting service
- per time interval in KSM. Those keys are sent to delete metadata and
- generate transactions in SCM for next async deletion between SCM
- and DataNode.
- </description>
- </property>
- <property>
- <name>ozone.ksm.address</name>
- <value/>
- <tag>KSM, REQUIRED</tag>
- <description>
- The address of the Ozone KSM service. This allows clients to discover
- the KSMs address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.group.rights</name>
- <value>READ_WRITE</value>
- <tag>KSM, SECURITY</tag>
- <description>
- Default group permissions in Ozone KSM.
- </description>
- </property>
- <property>
- <name>ozone.ksm.handler.count.key</name>
- <value>20</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- The number of RPC handler threads for KSM service endpoints.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http-address</name>
- <value>0.0.0.0:9874</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The address and the base port where the KSM web UI will listen on.
-
- If the port is 0, then the server will start on a free port. However, it
- is best to specify a well-known port, so it is easy to connect and see
- the KSM management UI.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http-bind-host</name>
- <value>0.0.0.0</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The actual address the KSM web server will bind to. If this optional
- the address is set, it overrides only the hostname portion of
- ozone.ksm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.http.enabled</name>
- <value>true</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- Property to enable or disable KSM web user interface.
- </description>
- </property>
- <property>
- <name>ozone.ksm.https-address</name>
- <value>0.0.0.0:9875</value>
- <tag>KSM, MANAGEMENT, SECURITY</tag>
- <description>
- The address and the base port where the KSM web UI will listen
- on using HTTPS.
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.ksm.https-bind-host</name>
- <value>0.0.0.0</value>
- <tag>KSM, MANAGEMENT, SECURITY</tag>
- <description>
- The actual address the KSM web server will bind to using HTTPS.
- If this optional address is set, it overrides only the hostname portion of
- ozone.ksm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.ksm.keytab.file</name>
- <value/>
- <tag>KSM, SECURITY</tag>
- <description>
- The keytab file for Kerberos authentication in KSM.
- </description>
- </property>
- <property>
- <name>ozone.ksm.db.cache.size.mb</name>
- <value>128</value>
- <tag>KSM, PERFORMANCE</tag>
- <description>
- The size of KSM DB cache in MB that used for caching files.
- This value is set to an abnormally low value in the default configuration.
- That is to make unit testing easy. Generally, this value should be set to
- something like 16GB or more, if you intend to use Ozone at scale.
-
- A large value for this key allows a proportionally larger amount of KSM
- metadata to be cached in memory. This makes KSM operations faster.
- </description>
- </property>
- <property>
- <name>ozone.ksm.user.max.volume</name>
- <value>1024</value>
- <tag>KSM, MANAGEMENT</tag>
- <description>
- The maximum number of volumes a user can have on a cluster.Increasing or
- decreasing this number has no real impact on ozone cluster. This is
- defined only for operational purposes. Only an administrator can create a
- volume, once a volume is created there are no restrictions on the number
- of buckets or keys inside each bucket a user can create.
- </description>
- </property>
- <property>
- <name>ozone.ksm.user.rights</name>
- <value>READ_WRITE</value>
- <tag>KSM, SECURITY</tag>
- <description>
- Default user permissions used in KSM.
- </description>
- </property>
- <property>
- <name>ozone.localstorage.root</name>
- <value>${hadoop.tmp.dir}/ozone</value>
- <tag>OZONE, DEBUG</tag>
- <description>
- This is used only for testing purposes. This value is used by the local
- storage handler to simulate a REST backend. This is useful only when
- debugging the REST front end independent of KSM and SCM. To be removed.
- </description>
- </property>
- <property>
- <name>ozone.metadata.dirs</name>
- <value/>
- <tag>OZONE, KSM, SCM, CONTAINER, REQUIRED, STORAGE</tag>
- <description>
- Ozone metadata is shared among KSM, which acts as the namespace
- manager for ozone, SCM which acts as the block manager and data nodes
- which maintain the name of the key(Key Name and BlockIDs). This
- replicated and distributed metadata store is maintained under the
- directory pointed by this key. Since metadata can be I/O intensive, at
- least on KSM and SCM we recommend having SSDs. If you have the luxury
- of mapping this path to SSDs on all machines in the cluster, that will
- be excellent.
-
- If Ratis metadata directories are not specified, Ratis server will emit a
- warning and use this path for storing its metadata too.
- </description>
- </property>
- <property>
- <name>ozone.metastore.impl</name>
- <value>RocksDB</value>
- <tag>OZONE, KSM, SCM, CONTAINER, STORAGE</tag>
- <description>
- Ozone metadata store implementation. Ozone metadata are well
- distributed to multiple services such as ksm, scm. They are stored in
- some local key-value databases. This property determines which database
- library to use. Supported value is either LevelDB or RocksDB.
- </description>
- </property>
-
- <property>
- <name>ozone.metastore.rocksdb.statistics</name>
- <value>ALL</value>
- <tag>OZONE, KSM, SCM, STORAGE, PERFORMANCE</tag>
- <description>
- The statistics level of the rocksdb store. If you use any value from
- org.rocksdb.StatsLevel (eg. ALL or EXCEPT_DETAILED_TIMERS), the rocksdb
- statistics will be exposed over JMX bean with the choosed setting. Set
- it to OFF to not initialize rocksdb statistics at all. Please note that
- collection of statistics could have 5-10% performance penalty.
- Check the rocksdb documentation for more details.
- </description>
- </property>
-
- <property>
- <name>ozone.scm.block.client.address</name>
- <value/>
- <tag>OZONE, SCM</tag>
- <description>The address of the Ozone SCM block client service. If not
- defined value of ozone.scm.client.address is used.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.client.bind.host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, SCM</tag>
- <description>
- The hostname or IP address used by the SCM block client
- endpoint to bind.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.client.port</name>
- <value>9863</value>
- <tag>OZONE, SCM</tag>
- <description>
- The port number of the Ozone SCM block client service.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.deletion.max.retry</name>
- <value>4096</value>
- <tag>OZONE, SCM</tag>
- <description>
- SCM wraps up many blocks in a deletion transaction and sends that to data
- node for physical deletion periodically. This property determines how many
- times SCM is going to retry sending a deletion operation to the data node.
- </description>
- </property>
- <property>
- <name>ozone.scm.block.size.in.mb</name>
- <value>256</value>
- <tag>OZONE, SCM</tag>
- <description>
- The default size of a scm block in bytes. This is maps to the default
- Ozone block size.
- </description>
- </property>
- <property>
- <name>ozone.scm.chunk.size</name>
- <value>16777216</value>
- <tag>OZONE, SCM, CONTAINER, PERFORMANCE</tag>
- <description>
- The chunk size for reading/writing chunk operations in bytes.
-
- The chunk size defaults to 8MB. If the value configured is more than the
- maximum size (16MB), it will be reset to the maximum size. This maps to
- the network packet sizes and file write operations in the client to
- datanode protocol.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.address</name>
- <value/>
- <tag>OZONE, SCM, REQUIRED</tag>
- <description>
- The address of the Ozone SCM client service. This is a required setting.
-
- It is a string in the host:port format. The port number is optional
- and defaults to 9860.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.bind.host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, SCM, MANAGEMENT</tag>
- <description>The hostname or IP address used by the SCM client endpoint to
- bind.
- This setting is used by the SCM only and never used by clients.
-
- The setting can be useful in multi-homed setups to restrict the
- availability of the SCM client service to a specific interface.
-
- The default is appropriate for most clusters.
- </description>
- </property>
- <property>
- <name>ozone.scm.client.port</name>
- <value>9860</value>
- <tag>OZONE, SCM, MANAGEMENT</tag>
- <description>The port number of the Ozone SCM client service.</description>
- </property>
- <property>
- <name>ozone.scm.container.deletion-choosing.policy</name>
- <value>
- org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
- </value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The policy used for choosing desire containers for block deletion.
- Datanode selects some containers to process block deletion
- in a certain interval defined by ozone.block.deleting.service.interval.
- The number of containers to process in each interval is defined
- by ozone.block.deleting.container.limit.per.interval. This property is
- used to configure the policy applied while selecting containers.
- There are two policies supporting now:
- RandomContainerDeletionChoosingPolicy and
- TopNOrderedContainerDeletionChoosingPolicy.
- org.apache.hadoop.ozone.container.common.impl.RandomContainerDeletionChoosingPolicy
- implements a simply random policy that to return a random list of
- containers.
- org.apache.hadoop.ozone.container.common.impl.TopNOrderedContainerDeletionChoosingPolicy
- implements a policy that choosing top count number of containers in a
- pending-deletion-blocks's num
- based descending order.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.placement.impl</name>
- <value>
- org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom
- </value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>Placement policy class for containers.
- Defaults to SCMContainerPlacementRandom.class
- </description>
- </property>
- <property>
- <name>ozone.scm.container.provision_batch_size</name>
- <value>20</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Pre-provision specified number of containers for block
- allocation.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.report.processing.interval</name>
- <value>60s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>Time interval for scm to process container reports
- for a node pool. Scm handles node pool reports in a cyclic clock
- manner, it fetches pools periodically with this time interval.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.reports.wait.timeout</name>
- <value>300s</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>Maximum time to wait in seconds for processing all container
- reports from
- a node pool. It determines the timeout for a
- node pool report.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.size.gb</name>
- <value>5</value>
- <tag>OZONE, PERFORMANCE, MANAGEMENT</tag>
- <description>
- Default container size used by Ozone. This value is specified
- in GB.
- There are two considerations while picking this number. The speed at which
- a container can be replicated, determined by the network speed and the
- metadata that each container generates. So selecting a large number
- creates less SCM metadata, but recovery time will be more. 5GB is a number
- that maps to quick replication times in gigabit networks, but still
- balances the amount of metadata.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.address</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address of the Ozone SCM service used for internal
- communication between the DataNodes and the SCM.
-
- It is a string in the host:port format. The port number is optional
- and defaults to 9861.
-
- This setting is optional. If unspecified then the hostname portion
- is picked from the ozone.scm.client.address setting and the
- default service port of 9861 is chosen.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.bind.host</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The hostname or IP address used by the SCM service endpoint to
- bind.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.id</name>
- <value/>
- <tag>OZONE, MANAGEMENT</tag>
- <description>The path that datanodes will use to store the datanode ID.
- If this value is not set, then datanode ID is created under the
- metadata directory.
- </description>
- </property>
- <property>
- <name>ozone.scm.datanode.port</name>
- <value>9861</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The port number of the Ozone SCM service.
- </description>
- </property>
- <property>
- <name>ozone.scm.db.cache.size.mb</name>
- <value>128</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>SCM keeps track of the Containers in the cluster. This DB holds
- the container metadata. This value is set to a small value to make the
- unit
- testing runs smooth. In production, we recommend a value of 16GB or
- higher. This allows SCM to avoid disk I/O's while looking up the container
- location.
- </description>
- </property>
- <property>
- <name>ozone.scm.dead.node.interval</name>
- <value>10m</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The interval between heartbeats before a node is tagged as dead.
- </description>
- </property>
- <property>
- <name>ozone.scm.handler.count.key</name>
- <value>10</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- The number of RPC handler threads for each SCM service
- endpoint.
-
- The default is appropriate for small clusters (tens of nodes).
-
- Set a value that is appropriate for the cluster size. Generally, HDFS
- recommends RPC handler count is set to 20 * log2(Cluster Size) with an
- upper limit of 200. However, SCM will not have the same amount of
- traffic as Namenode, so a value much smaller than that will work well too.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.interval</name>
- <value>30s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The heartbeat interval from a data node to SCM. Yes,
- it is not three but 30, since most data nodes will heart beating via Ratis
- heartbeats. If a client is not able to talk to a data node, it will notify
- KSM/SCM eventually. So a 30 second HB seems to work. This assumes that
- replication strategy used is Ratis if not, this value should be set to
- something smaller like 3 seconds.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.log.warn.interval.count</name>
- <value>10</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Defines how frequently we will log the missing of a heartbeat to SCM.
- For example in the default case, we will write a warning message for each
- ten consecutive heartbeats that we miss to SCM. This helps in reducing
- clutter in a data node log, but trade off is that logs will have less of
- this statement.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.rpc-timeout</name>
- <value>1000</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Timeout value for the RPC from Datanode to SCM in milliseconds.
- </description>
- </property>
- <property>
- <name>ozone.scm.heartbeat.thread.interval</name>
- <value>3s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- When a heartbeat from the data node arrives on SCM, It is queued for
- processing with the time stamp of when the heartbeat arrived. There is a
- heartbeat processing thread inside SCM that runs at a specified interval.
- This value controls how frequently this thread is run.
-
- There are some assumptions build into SCM such as this value should allow
- the heartbeat processing thread to run at least three times more
- frequently than heartbeats and at least five times more than stale node
- detection time. If you specify a wrong value, SCM will gracefully refuse
- to run. For more info look at the node manager tests in SCM.
-
- In short, you don't need to change this.
- </description>
- </property>
- <property>
- <name>ozone.scm.http-address</name>
- <value>0.0.0.0:9876</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address and the base port where the SCM web ui will listen on.
-
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.scm.http-bind-host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The actual address the SCM web server will bind to. If this
- optional address is set, it overrides only the hostname portion of
- ozone.scm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.scm.http.enabled</name>
- <value>true</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- Property to enable or disable SCM web ui.
- </description>
- </property>
- <property>
- <name>ozone.scm.https-address</name>
- <value>0.0.0.0:9877</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The address and the base port where the SCM web UI will listen
- on using HTTPS.
-
- If the port is 0 then the server will start on a free port.
- </description>
- </property>
- <property>
- <name>ozone.scm.https-bind-host</name>
- <value>0.0.0.0</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The actual address the SCM web server will bind to using HTTPS.
- If this optional address is set, it overrides only the hostname portion of
- ozone.scm.http-address.
- </description>
- </property>
- <property>
- <name>ozone.scm.keytab.file</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>
- The keytab file for Kerberos authentication in SCM.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.container.report.threads</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Maximum number of threads to process container reports in scm.
- Each container report from a data node is processed by scm in a worker
- thread, fetched from a thread pool. This property is used to control the
- maximum size of the thread pool.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.hb.count.to.process</name>
- <value>5000</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- The maximum number of heartbeat to process per loop of the
- heartbeat process thread. Please see
- ozone.scm.heartbeat.thread.interval
- for more info.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, MANAGEMENT, PERFORMANCE</tag>
- <description>
- Number of node pools to process in parallel.
- </description>
- </property>
- <property>
- <name>ozone.scm.names</name>
- <value/>
- <tag>OZONE</tag>
- <description>
- The value of this property is a set of DNS | DNS:PORT | IP
- Address | IP:PORT. Written as a comma separated string. e.g. scm1,
- scm2:8020, 7.7.7.7:7777.
- This property allows datanodes to discover where SCM is, so that
- datanodes can send heartbeat to SCM.
- </description>
- </property>
- <property>
- <name>ozone.scm.stale.node.interval</name>
- <value>90s</value>
- <tag>OZONE, MANAGEMENT</tag>
- <description>
- The interval for stale node flagging. Please
- see ozone.scm.heartbeat.thread.interval before changing this value.
- </description>
- </property>
- <property>
- <name>ozone.scm.max.nodepool.processing.threads</name>
- <value>1</value>
- <tag>OZONE, SCM</tag>
- <description>
- Controls the number of node pools that can be processed in parallel by
- Container Supervisor.
- </description>
- </property>
- <property>
- <name>ozone.trace.enabled</name>
- <value>false</value>
- <tag>OZONE, DEBUG</tag>
- <description>
- Setting this flag to true dumps the HTTP request/ response in
- the logs. Very useful when debugging REST protocol.
- </description>
- </property>
- <property>
- <name>ozone.web.authentication.kerberos.principal</name>
- <value/>
- <tag>OZONE, SECURITY</tag>
- <description>
- The server principal used by the SCM and KSM for web UI SPNEGO
- authentication when Kerberos security is enabled. This is typically set to
- HTTP/_HOST@REALM.TLD The SPNEGO server principal begins with the prefix
- HTTP/ by convention.
-
- If the value is '*', the web server will attempt to login with
- every principal specified in the keytab file.
- </description>
- </property>
-
- <!--Client Settings-->
- <property>
- <name>scm.container.client.idle.threshold</name>
- <value>10s</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- In the standalone pipelines, the SCM clients use netty to
- communicate with the container. It also uses connection pooling to
- reduce client side overheads. This allows a connection to stay idle for
- a while before the connection is closed.
- </description>
- </property>
- <property>
- <name>scm.container.client.max.size</name>
- <value>256</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Controls the maximum number of connections that we cached via
- clientconnection pooling. If the number of connection
- exceed this count then the oldest idle connection is evicted.
- </description>
- </property>
-
- <property>
- <name>scm.container.client.max.outstanding.requests</name>
- <value>100</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Controls the maximum number of outstanding async requests that can be
- handled by the Standalone as well as Ratis client.
- </description>
- </property>
-
- <property>
- <name>ozone.scm.container.creation.lease.timeout</name>
- <value>60s</value>
- <tag>OZONE, SCM</tag>
- <description>
- Container creation timeout in milliseconds to be used by SCM. When
- BEGIN_CREATE event happens the container is moved from ALLOCATED to
- CREATING state, SCM will now wait for the configured amount of time
- to get COMPLETE_CREATE event if it doesn't receive it will move the
- container to DELETING.
- </description>
- </property>
-
- <property>
- <name>ozone.key.preallocation.maxsize</name>
- <value>134217728</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- When a new key write request is sent to KSM, if a size is requested, at most
- 128MB of size is allocated at request time. If client needs more space for the
- write, separate block allocation requests will be made.
- </description>
- </property>
-
- <property>
- <name>ozone.client.list.cache</name>
- <value>1000</value>
- <tag>OZONE, PERFORMANCE</tag>
- <description>
- Configuration property to configure the cache size of client list calls.
- </description>
- </property>
-
- <property>
- <name>ozone.replication</name>
- <value>3</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- Default replication value. The actual number of replications can be
- specified when writing the key. The default is used if replication
- is not specified. Supported values: 1 and 3.
- </description>
- </property>
-
- <property>
- <name>ozone.replication.type</name>
- <value>RATIS</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- Default replication type to be used while writing key into ozone. The
- value can be specified when writing the key, default is used when
- nothing is specified. Supported values: RATIS, STAND_ALONE and CHAINED.
- </description>
- </property>
- <property>
- <name>ozone.scm.container.close.threshold</name>
- <value>0.9f</value>
- <tag>OZONE, SCM</tag>
- <description>
- This determines the threshold to be used for closing a container.
- When the container used percentage reaches this threshold,
- the container will be closed. Value should be a positive, non-zero
- percentage in float notation (X.Yf), with 1.0f meaning 100%.
- </description>
- </property>
- <property>
- <name>ozone.rest.client.http.connection.max</name>
- <value>100</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- This defines the overall connection limit for the connection pool used in
- RestClient.
- </description>
- </property>
- <property>
- <name>ozone.rest.client.http.connection.per-route.max</name>
- <value>20</value>
- <tag>OZONE, CLIENT</tag>
- <description>
- This defines the connection limit per one HTTP route/host. Total max
- connection is limited by ozone.rest.client.http.connection.max property.
- </description>
- </property>
-
- <property>
- <name>ozone.open.key.cleanup.service.interval.seconds</name>
- <value>86400</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- A background job periodically checks open key entries and delete the expired ones. This entry controls the
- interval of this cleanup check.
- </description>
- </property>
-
- <property>
- <name>ozone.open.key.expire.threshold</name>
- <value>86400</value>
- <tag>OZONE, KSM, PERFORMANCE</tag>
- <description>
- Controls how long an open key operation is considered active. Specifically, if a key
- has been open longer than the value of this config entry, that open key is considered as
- expired (e.g. due to client crash). Default to 24 hours.
- </description>
- </property>
-</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
index 3b67167..9e2732c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/datanode/dn.js
@@ -18,11 +18,11 @@
(function () {
"use strict";
- var data = {ozone: {enabled: false}};
+ var data = {};
dust.loadSource(dust.compile($('#tmpl-dn').html(), 'dn'));
- function loadDatanodeInfo() {
+ function load() {
$.get('/jmx?qry=Hadoop:service=DataNode,name=DataNodeInfo', function(resp) {
data.dn = workaround(resp.beans[0]);
data.dn.HostName = resp.beans[0]['DatanodeHostname'];
@@ -30,26 +30,6 @@
}).fail(show_err_msg);
}
- function loadOzoneScmInfo() {
- $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=SCMConnectionManager', function (resp) {
- if (resp.beans.length > 0) {
- data.ozone.SCMServers = resp.beans[0].SCMServers;
- data.ozone.enabled = true;
- render();
- }
- }).fail(show_err_msg);
- }
-
- function loadOzoneStorageInfo() {
- $.get('/jmx?qry=Hadoop:service=OzoneDataNode,name=ContainerLocationManager', function (resp) {
- if (resp.beans.length > 0) {
- data.ozone.LocationReport = resp.beans[0].LocationReport;
- data.ozone.enabled = true;
- render();
- }
- }).fail(show_err_msg);
- }
-
function workaround(dn) {
function node_map_to_array(nodes) {
var res = [];
@@ -85,8 +65,6 @@
$('#alert-panel').show();
}
- loadDatanodeInfo();
- loadOzoneScmInfo();
- loadOzoneStorageInfo();
+ load();
})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
deleted file mode 100644
index bee42bf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/index.html
+++ /dev/null
@@ -1,70 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <meta name="description" content="HDFS Key Space Manager">
-
- <title>HDFS Key Space Manager</title>
-
- <link href="static/bootstrap-3.0.2/css/bootstrap.min.css" rel="stylesheet">
- <link href="static/hadoop.css" rel="stylesheet">
- <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
- <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="ksm">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
- aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="#">HDFS KSM</a>
- </div>
- <navmenu
- metrics="{ 'Ksm metrics' : '#!/metrics/ksm', 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
- </div>
-</header>
-
-<div class="container-fluid">
- <ng-view></ng-view>
-</div><!-- /.container -->
-
-<script src="static/jquery-1.10.2.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="ksm.js"></script>
-<script src="static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
deleted file mode 100644
index e63fb00..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm-metrics.html
+++ /dev/null
@@ -1,44 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h1>KSM Metrics</h1>
-
-<div ng-repeat="(type,numbers) in $ctrl.metrics.nums">
- <h2>{{type}}</h2>
- <div class="container">
- <div class="col-md-6">
- <h3>Requests ({{numbers.ops}} ops)</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.all"></nvd3>
- </div>
- <div class="col-md-6">
- <h3>Failures</h3>
- <nvd3 options="$ctrl.graphOptions"
- data="numbers.failures"></nvd3>
- </div>
- </div>
-</div>
-
-<div ng-show="$ctrl.metrics.others.length > 0">
- <h2>Other JMX properties</h2>
-
- <table class="table">
- <tr ng-repeat="metric in $ctrl.metrics.others">
- <td>{{metric.key}}</td>
- <td>{{metric.value}}</td>
- </tr>
- </table>
-</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
deleted file mode 100644
index 7fb52b1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/ksm.js
+++ /dev/null
@@ -1,108 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- var isIgnoredJmxKeys = function (key) {
- return key == 'name' || key == 'modelerType' || key.match(/tag.*/);
- };
-
- angular.module('ksm', ['ozone', 'nvd3']);
- angular.module('ksm').config(function ($routeProvider) {
- $routeProvider
- .when("/metrics/ksm", {
- template: "<ksm-metrics></ksm-metrics>"
- });
- });
- angular.module('ksm').component('ksmMetrics', {
- templateUrl: 'ksm-metrics.html',
- controller: function ($http) {
- var ctrl = this;
-
- ctrl.graphOptions = {
- chart: {
- type: 'pieChart',
- height: 500,
- x: function (d) {
- return d.key;
- },
- y: function (d) {
- return d.value;
- },
- showLabels: true,
- labelType: 'value',
- duration: 500,
- labelThreshold: 0.01,
- labelSunbeamLayout: true,
- legend: {
- margin: {
- top: 5,
- right: 35,
- bottom: 5,
- left: 0
- }
- }
- }
- };
-
-
- $http.get("jmx?qry=Hadoop:service=KeySpaceManager,name=KSMMetrics")
- .then(function (result) {
-
- var groupedMetrics = {others: [], nums: {}};
- var metrics = result.data.beans[0]
- for (var key in metrics) {
- var numericalStatistic = key.match(/Num([A-Z][a-z]+)(.+?)(Fails)?$/);
- if (numericalStatistic) {
- var type = numericalStatistic[1];
- var name = numericalStatistic[2];
- var failed = numericalStatistic[3];
- groupedMetrics.nums[type] = groupedMetrics.nums[type] || {
- failures: [],
- all: []
- };
- if (failed) {
- groupedMetrics.nums[type].failures.push({
- key: name,
- value: metrics[key]
- })
- } else {
- if (name == "Ops") {
- groupedMetrics.nums[type].ops = metrics[key]
- } else {
- groupedMetrics.nums[type].all.push({
- key: name,
- value: metrics[key]
- })
- }
- }
- } else if (isIgnoredJmxKeys(key)) {
- //ignore
- } else {
- groupedMetrics.others.push({
- 'key': key,
- 'value': metrics[key]
- });
- }
- }
- ctrl.metrics = groupedMetrics;
- })
- }
- });
-
-})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
deleted file mode 100644
index e442adc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.css
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-body {
- padding-top: 50px;
-}
-.starter-template {
- padding: 40px 15px;
- text-align: center;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
deleted file mode 100644
index 0821899..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/ksm/main.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<overview>
-</overview>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
deleted file mode 100644
index 3407f51..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/index.html
+++ /dev/null
@@ -1,76 +0,0 @@
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1">
- <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
- <meta name="description" content="HDFS Storage Container Manager">
-
- <title>HDFS Storage Container Manager</title>
-
- <link href="static/bootstrap-3.0.2/css/bootstrap.min.css" rel="stylesheet">
- <link href="static/hadoop.css" rel="stylesheet">
- <link href="static/nvd3-1.8.5.min.css" rel="stylesheet">
-
- <link href="static/ozone.css" rel="stylesheet">
-
-</head>
-
-<body ng-app="scm">
-
-<header class="navbar navbar-inverse navbar-fixed-top bs-docs-nav">
- <div class="container-fluid">
- <div class="navbar-header">
- <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar"
- aria-expanded="false" aria-controls="navbar">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
- <a class="navbar-brand" href="#">HDFS SCM</a>
- </div>
-
-
- <navmenu
- metrics="{ 'Rpc metrics' : '#!/metrics/rpc'}"></navmenu>
-
-
- </div>
-</header>
-
-<div class="container-fluid" style="margin: 12pt">
-
- <ng-view></ng-view>
-
-</div><!-- /.container -->
-
-<script src="static/jquery-1.10.2.min.js"></script>
-<script src="static/angular-1.6.4.min.js"></script>
-<script src="static/angular-route-1.6.4.min.js"></script>
-<script src="static/d3-3.5.17.min.js"></script>
-<script src="static/nvd3-1.8.5.min.js"></script>
-<script src="static/angular-nvd3-1.0.9.min.js"></script>
-<script src="static/ozone.js"></script>
-<script src="scm.js"></script>
-<script src="static/bootstrap-3.0.2/js/bootstrap.min.js"></script>
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
deleted file mode 100644
index 2666f81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/main.html
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-<overview>
- <scm-overview>
- </scm-overview>
-</overview>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
deleted file mode 100644
index fca23ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm-overview.html
+++ /dev/null
@@ -1,60 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h2>Node counts</h2>
-
-<table class="table table-bordered table-striped" class="col-md-6">
- <tbody>
- <tr ng-repeat="typestat in $ctrl.nodemanagermetrics.NodeCount | orderBy:'key':false:$ctrl.nodeOrder">
- <td>{{typestat.key}}</td>
- <td>{{typestat.value}}</td>
- </tr>
- </tbody>
-</table>
-
-<h2>Status</h2>
-<table class="table table-bordered table-striped" class="col-md-6">
- <tbody>
- <tr>
- <td>Client Rpc port</td>
- <td>{{$ctrl.overview.jmx.ClientRpcPort}}</td>
- </tr>
- <tr>
- <td>Datanode Rpc port</td>
- <td>{{$ctrl.overview.jmx.DatanodeRpcPort}}</td>
- </tr>
- <tr>
- <td>Block Manager: Open containers</td>
- <td>{{$ctrl.blockmanagermetrics.OpenContainersNo}}</td>
- </tr>
- <tr>
- <td>Node Manager: Minimum chill mode nodes</td>
- <td>{{$ctrl.nodemanagermetrics.MinimumChillModeNodes}}</td>
- </tr>
- <tr>
- <td>Node Manager: Out-of-node chill mode</td>
- <td>{{$ctrl.nodemanagermetrics.OutOfNodeChillMode}}</td>
- </tr>
- <tr>
- <td>Node Manager: Chill mode status</td>
- <td>{{$ctrl.nodemanagermetrics.ChillModeStatus}}</td>
- </tr>
- <tr>
- <td>Node Manager: Manual chill mode</td>
- <td>{{$ctrl.nodemanagermetrics.InManualChillMode}}</td>
- </tr>
- </tbody>
-</table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
deleted file mode 100644
index bcfa8b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/scm/scm.js
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
- angular.module('scm', ['ozone', 'nvd3']);
-
- angular.module('scm').component('scmOverview', {
- templateUrl: 'scm-overview.html',
- require: {
- overview: "^overview"
- },
- controller: function ($http) {
- var ctrl = this;
- $http.get("jmx?qry=Hadoop:service=BlockManager,name=*")
- .then(function (result) {
- ctrl.blockmanagermetrics = result.data.beans[0];
- });
- $http.get("jmx?qry=Hadoop:service=SCMNodeManager,name=SCMNodeManagerInfo")
- .then(function (result) {
- ctrl.nodemanagermetrics = result.data.beans[0];
- });
-
- var statusSortOrder = {
- "HEALTHY": "a",
- "STALE": "b",
- "DEAD": "c",
- "UNKNOWN": "z",
- "DECOMMISSIONING": "x",
- "DECOMMISSIONED": "y"
- };
- ctrl.nodeOrder = function (v1, v2) {
- //status with non defined sort order will be "undefined"
- return ("" + statusSortOrder[v1.value]).localeCompare("" + statusSortOrder[v2.value])
- }
-
- }
- });
-
-})();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
deleted file mode 100644
index 833d1a8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/InProgressPool.java
+++ /dev/null
@@ -1,313 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.NodePoolManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.Predicate;
-import java.util.stream.Collectors;
-
-import static com.google.common.util.concurrent.Uninterruptibles
- .sleepUninterruptibly;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
- .NodeState.HEALTHY;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
- .NodeState.STALE;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
- .NodeState.UNKNOWN;
-
-/**
- * These are pools that are actively checking for replication status of the
- * containers.
- */
-public final class InProgressPool {
- public static final Logger LOG =
- LoggerFactory.getLogger(InProgressPool.class);
-
- private final PeriodicPool pool;
- private final NodeManager nodeManager;
- private final NodePoolManager poolManager;
- private final ExecutorService executorService;
- private final Map<String, Integer> containerCountMap;
- private final Map<String, Boolean> processedNodeSet;
- private final long startTime;
- private ProgressStatus status;
- private AtomicInteger nodeCount;
- private AtomicInteger nodeProcessed;
- private AtomicInteger containerProcessedCount;
- private long maxWaitTime;
- /**
- * Constructs an pool that is being processed.
- * @param maxWaitTime - Maximum wait time in milliseconds.
- * @param pool - Pool that we are working against
- * @param nodeManager - Nodemanager
- * @param poolManager - pool manager
- * @param executorService - Shared Executor service.
- */
- InProgressPool(long maxWaitTime, PeriodicPool pool,
- NodeManager nodeManager, NodePoolManager poolManager,
- ExecutorService executorService) {
- Preconditions.checkNotNull(pool);
- Preconditions.checkNotNull(nodeManager);
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(executorService);
- Preconditions.checkArgument(maxWaitTime > 0);
- this.pool = pool;
- this.nodeManager = nodeManager;
- this.poolManager = poolManager;
- this.executorService = executorService;
- this.containerCountMap = new ConcurrentHashMap<>();
- this.processedNodeSet = new ConcurrentHashMap<>();
- this.maxWaitTime = maxWaitTime;
- startTime = Time.monotonicNow();
- }
-
- /**
- * Returns periodic pool.
- *
- * @return PeriodicPool
- */
- public PeriodicPool getPool() {
- return pool;
- }
-
- /**
- * We are done if we have got reports from all nodes or we have
- * done waiting for the specified time.
- *
- * @return true if we are done, false otherwise.
- */
- public boolean isDone() {
- return (nodeCount.get() == nodeProcessed.get()) ||
- (this.startTime + this.maxWaitTime) > Time.monotonicNow();
- }
-
- /**
- * Gets the number of containers processed.
- *
- * @return int
- */
- public int getContainerProcessedCount() {
- return containerProcessedCount.get();
- }
-
- /**
- * Returns the start time in milliseconds.
- *
- * @return - Start Time.
- */
- public long getStartTime() {
- return startTime;
- }
-
- /**
- * Get the number of nodes in this pool.
- *
- * @return - node count
- */
- public int getNodeCount() {
- return nodeCount.get();
- }
-
- /**
- * Get the number of nodes that we have already processed container reports
- * from.
- *
- * @return - Processed count.
- */
- public int getNodeProcessed() {
- return nodeProcessed.get();
- }
-
- /**
- * Returns the current status.
- *
- * @return Status
- */
- public ProgressStatus getStatus() {
- return status;
- }
-
- /**
- * Starts the reconciliation process for all the nodes in the pool.
- */
- public void startReconciliation() {
- List<DatanodeID> datanodeIDList =
- this.poolManager.getNodes(pool.getPoolName());
- if (datanodeIDList.size() == 0) {
- LOG.error("Datanode list for {} is Empty. Pool with no nodes ? ",
- pool.getPoolName());
- this.status = ProgressStatus.Error;
- return;
- }
-
- nodeProcessed = new AtomicInteger(0);
- containerProcessedCount = new AtomicInteger(0);
- nodeCount = new AtomicInteger(0);
- /*
- Ask each datanode to send us commands.
- */
- SendContainerCommand cmd = SendContainerCommand.newBuilder().build();
- for (DatanodeID id : datanodeIDList) {
- NodeState currentState = getNodestate(id);
- if (currentState == HEALTHY || currentState == STALE) {
- nodeCount.incrementAndGet();
- // Queue commands to all datanodes in this pool to send us container
- // report. Since we ignore dead nodes, it is possible that we would have
- // over replicated the container if the node comes back.
- nodeManager.addDatanodeCommand(id, cmd);
- }
- }
- this.status = ProgressStatus.InProgress;
- this.getPool().setLastProcessedTime(Time.monotonicNow());
- }
-
- /**
- * Gets the node state.
- *
- * @param id - datanode ID.
- * @return NodeState.
- */
- private NodeState getNodestate(DatanodeID id) {
- NodeState currentState = UNKNOWN;
- int maxTry = 100;
- // We need to loop to make sure that we will retry if we get
- // node state unknown. This can lead to infinite loop if we send
- // in unknown node ID. So max try count is used to prevent it.
-
- int currentTry = 0;
- while (currentState == UNKNOWN && currentTry < maxTry) {
- // Retry to make sure that we deal with the case of node state not
- // known.
- currentState = nodeManager.getNodeState(id);
- currentTry++;
- if (currentState == UNKNOWN) {
- // Sleep to make sure that this is not a tight loop.
- sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
- }
- }
- if (currentState == UNKNOWN) {
- LOG.error("Not able to determine the state of Node: {}, Exceeded max " +
- "try and node manager returns UNKNOWN state. This indicates we " +
- "are dealing with a node that we don't know about.", id);
- }
- return currentState;
- }
-
- /**
- * Queues a container Report for handling. This is done in a worker thread
- * since decoding a container report might be compute intensive . We don't
- * want to block since we have asked for bunch of container reports
- * from a set of datanodes.
- *
- * @param containerReport - ContainerReport
- */
- public void handleContainerReport(
- ContainerReportsRequestProto containerReport) {
- if (status == ProgressStatus.InProgress) {
- executorService.submit(processContainerReport(containerReport));
- } else {
- LOG.debug("Cannot handle container report when the pool is in {} status.",
- status);
- }
- }
-
- private Runnable processContainerReport(
- ContainerReportsRequestProto reports) {
- return () -> {
- DatanodeID datanodeID =
- DatanodeID.getFromProtoBuf(reports.getDatanodeID());
- if (processedNodeSet.computeIfAbsent(datanodeID.getDatanodeUuid(),
- (k) -> true)) {
- nodeProcessed.incrementAndGet();
- LOG.debug("Total Nodes processed : {} Node Name: {} ", nodeProcessed,
- datanodeID.getDatanodeUuid());
- for (ContainerInfo info : reports.getReportsList()) {
- containerProcessedCount.incrementAndGet();
- LOG.debug("Total Containers processed: {} Container Name: {}",
- containerProcessedCount.get(), info.getContainerName());
-
- // Update the container map with count + 1 if the key exists or
- // update the map with 1. Since this is a concurrentMap the
- // computation and update is atomic.
- containerCountMap.merge(info.getContainerName(), 1, Integer::sum);
- }
- }
- };
- }
-
- /**
- * Filter the containers based on specific rules.
- *
- * @param predicate -- Predicate to filter by
- * @return A list of map entries.
- */
- public List<Map.Entry<String, Integer>> filterContainer(
- Predicate<Map.Entry<String, Integer>> predicate) {
- return containerCountMap.entrySet().stream()
- .filter(predicate).collect(Collectors.toList());
- }
-
- /**
- * Used only for testing, calling this will abort container report
- * processing. This is very dangerous call and should not be made by any users
- */
- @VisibleForTesting
- public void setDoneProcessing() {
- nodeProcessed.set(nodeCount.get());
- }
-
- /**
- * Returns the pool name.
- *
- * @return Name of the pool.
- */
- String getPoolName() {
- return pool.getPoolName();
- }
-
- public void finalizeReconciliation() {
- status = ProgressStatus.Done;
- //TODO: Add finalizing logic. This is where actual reconciliation happens.
- }
-
- /**
- * Current status of the computing replication status.
- */
- public enum ProgressStatus {
- InProgress, Done, Error
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
deleted file mode 100644
index 35b1e76..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/PeriodicPool.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Periodic pool is a pool with a time stamp, this allows us to process pools
- * based on a cyclic clock.
- */
-public class PeriodicPool implements Comparable<PeriodicPool> {
- private final String poolName;
- private long lastProcessedTime;
- private AtomicLong totalProcessedCount;
-
- /**
- * Constructs a periodic pool.
- *
- * @param poolName - Name of the pool
- */
- public PeriodicPool(String poolName) {
- this.poolName = poolName;
- lastProcessedTime = 0;
- totalProcessedCount = new AtomicLong(0);
- }
-
- /**
- * Get pool Name.
- * @return PoolName
- */
- public String getPoolName() {
- return poolName;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(PeriodicPool o) {
- return Long.compare(this.lastProcessedTime, o.lastProcessedTime);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- PeriodicPool that = (PeriodicPool) o;
-
- return poolName.equals(that.poolName);
- }
-
- @Override
- public int hashCode() {
- return poolName.hashCode();
- }
-
- /**
- * Returns the Total Times we have processed this pool.
- *
- * @return processed count.
- */
- public long getTotalProcessedCount() {
- return totalProcessedCount.get();
- }
-
- /**
- * Gets the last time we processed this pool.
- * @return time in milliseconds
- */
- public long getLastProcessedTime() {
- return this.lastProcessedTime;
- }
-
-
- /**
- * Sets the last processed time.
- *
- * @param lastProcessedTime - Long in milliseconds.
- */
-
- public void setLastProcessedTime(long lastProcessedTime) {
- this.lastProcessedTime = lastProcessedTime;
- }
-
- /*
- * Increments the total processed count.
- */
- public void incTotalProcessedCount() {
- this.totalProcessedCount.incrementAndGet();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
deleted file mode 100644
index 82e4202..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-/*
- This package contains routines that manage replication of a container. This
- relies on container reports to understand the replication level of a
- container - UnderReplicated, Replicated, OverReplicated -- and manages the
- replication level based on that.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
deleted file mode 100644
index 4fb50e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/SCMException.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception thrown by SCM.
- */
-public class SCMException extends IOException {
- private final ResultCodes result;
-
- /**
- * Constructs an {@code IOException} with {@code null}
- * as its error detail message.
- */
- public SCMException(ResultCodes result) {
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- */
- public SCMException(String message, ResultCodes result) {
- super(message);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message
- * and cause.
- * <p>
- * <p> Note that the detail message associated with {@code cause} is
- * <i>not</i> automatically incorporated into this exception's detail
- * message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @since 1.6
- */
- public SCMException(String message, Throwable cause, ResultCodes result) {
- super(message, cause);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified cause and a
- * detail message of {@code (cause==null ? null : cause.toString())}
- * (which typically contains the class and detail message of {@code cause}).
- * This constructor is useful for IO exceptions that are little more
- * than wrappers for other throwables.
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @since 1.6
- */
- public SCMException(Throwable cause, ResultCodes result) {
- super(cause);
- this.result = result;
- }
-
- /**
- * Returns resultCode.
- * @return ResultCode
- */
- public ResultCodes getResult() {
- return result;
- }
-
- /**
- * Error codes to make it easy to decode these exceptions.
- */
- public enum ResultCodes {
- SUCCEESS,
- FAILED_TO_LOAD_NODEPOOL,
- FAILED_TO_FIND_NODE_IN_POOL,
- FAILED_TO_FIND_HEALTHY_NODES,
- FAILED_TO_FIND_NODES_WITH_SPACE,
- FAILED_TO_FIND_SUITABLE_NODE,
- INVALID_CAPACITY,
- INVALID_BLOCK_SIZE,
- CHILL_MODE_EXCEPTION,
- FAILED_TO_LOAD_OPEN_CONTAINER,
- FAILED_TO_ALLOCATE_CONTAINER,
- FAILED_TO_CHANGE_CONTAINER_STATE,
- CONTAINER_EXISTS,
- FAILED_TO_FIND_CONTAINER,
- FAILED_TO_FIND_CONTAINER_WITH_SPACE,
- BLOCK_EXISTS,
- FAILED_TO_FIND_BLOCK,
- IO_EXCEPTION,
- UNEXPECTED_CONTAINER_STATE,
- SCM_NOT_INITIALIZED
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
deleted file mode 100644
index 0922382..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/exceptions/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.exceptions;
-// Exceptions thrown by SCM.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
deleted file mode 100644
index bbf319b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/CommandQueue.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.util.Time;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Command Queue is queue of commands for the datanode.
- * <p>
- * Node manager, container Manager and key space managers can queue commands for
- * datanodes into this queue. These commands will be send in the order in which
- * there where queued.
- */
-public class CommandQueue {
- // This list is used as default return value.
- private static final List<SCMCommand> DEFAULT_LIST = new LinkedList<>();
- private final Map<DatanodeID, Commands> commandMap;
- private final Lock lock;
- private long commandsInQueue;
-
- /**
- * Returns number of commands in queue.
- * @return Command Count.
- */
- public long getCommandsInQueue() {
- return commandsInQueue;
- }
-
- /**
- * Constructs a Command Queue.
- * TODO : Add a flusher thread that throws away commands older than a certain
- * time period.
- */
- public CommandQueue() {
- commandMap = new HashMap<>();
- lock = new ReentrantLock();
- commandsInQueue = 0;
- }
-
- /**
- * This function is used only for test purposes.
- */
- @VisibleForTesting
- public void clear() {
- lock.lock();
- try {
- commandMap.clear();
- commandsInQueue = 0;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Returns a list of Commands for the datanode to execute, if we have no
- * commands returns a empty list otherwise the current set of
- * commands are returned and command map set to empty list again.
- *
- * @param datanodeID DatanodeID
- * @return List of SCM Commands.
- */
- @SuppressWarnings("unchecked")
- List<SCMCommand> getCommand(final DatanodeID datanodeID) {
- lock.lock();
- try {
- Commands cmds = commandMap.remove(datanodeID);
- List<SCMCommand> cmdList = null;
- if(cmds != null) {
- cmdList = cmds.getCommands();
- commandsInQueue -= cmdList.size() > 0 ? cmdList.size() : 0;
- // A post condition really.
- Preconditions.checkState(commandsInQueue >= 0);
- }
- return cmds == null ? DEFAULT_LIST : cmdList;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Adds a Command to the SCM Queue to send the command to container.
- *
- * @param datanodeID DatanodeID
- * @param command - Command
- */
- public void addCommand(final DatanodeID datanodeID, final SCMCommand
- command) {
- lock.lock();
- try {
- if (commandMap.containsKey(datanodeID)) {
- commandMap.get(datanodeID).add(command);
- } else {
- commandMap.put(datanodeID, new Commands(command));
- }
- commandsInQueue++;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Class that stores commands for a datanode.
- */
- private static class Commands {
- private long updateTime;
- private long readTime;
- private List<SCMCommand> commands;
-
- /**
- * Constructs a Commands class.
- */
- Commands() {
- commands = new LinkedList<>();
- updateTime = 0;
- readTime = 0;
- }
-
- /**
- * Creates the object and populates with the command.
- * @param command command to add to queue.
- */
- Commands(SCMCommand command) {
- this();
- this.add(command);
- }
-
- /**
- * Gets the last time the commands for this node was updated.
- * @return Time stamp
- */
- public long getUpdateTime() {
- return updateTime;
- }
-
- /**
- * Gets the last read time.
- * @return last time when these commands were read from this queue.
- */
- public long getReadTime() {
- return readTime;
- }
-
- /**
- * Adds a command to the list.
- *
- * @param command SCMCommand
- */
- public void add(SCMCommand command) {
- this.commands.add(command);
- updateTime = Time.monotonicNow();
- }
-
- /**
- * Returns the commands for this datanode.
- * @return command list.
- */
- public List<SCMCommand> getCommands() {
- List<SCMCommand> temp = this.commands;
- this.commands = new LinkedList<>();
- readTime = Time.monotonicNow();
- return temp;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
deleted file mode 100644
index 1e755b2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/HeartbeatQueueItem.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-/**
- * This class represents the item in SCM heartbeat queue.
- */
-public class HeartbeatQueueItem {
- private DatanodeID datanodeID;
- private long recvTimestamp;
- private SCMNodeReport nodeReport;
- private ReportState containerReportState;
-
- /**
- *
- * @param datanodeID - datanode ID of the heartbeat.
- * @param recvTimestamp - heartbeat receive timestamp.
- * @param nodeReport - node report associated with the heartbeat if any.
- * @param containerReportState - container report state.
- */
- HeartbeatQueueItem(DatanodeID datanodeID, long recvTimestamp,
- SCMNodeReport nodeReport, ReportState containerReportState) {
- this.datanodeID = datanodeID;
- this.recvTimestamp = recvTimestamp;
- this.nodeReport = nodeReport;
- this.containerReportState = containerReportState;
- }
-
- /**
- * @return datanode ID.
- */
- public DatanodeID getDatanodeID() {
- return datanodeID;
- }
-
- /**
- * @return node report.
- */
- public SCMNodeReport getNodeReport() {
- return nodeReport;
- }
-
- /**
- * @return container report state.
- */
- public ReportState getContainerReportState() {
- return containerReportState;
- }
-
- /**
- * @return heartbeat receive timestamp.
- */
- public long getRecvTimestamp() {
- return recvTimestamp;
- }
-
- /**
- * Builder for HeartbeatQueueItem.
- */
- public static class Builder {
- private DatanodeID datanodeID;
- private SCMNodeReport nodeReport;
- private ReportState containerReportState;
- private long recvTimestamp = monotonicNow();
-
- public Builder setDatanodeID(DatanodeID datanodeId) {
- this.datanodeID = datanodeId;
- return this;
- }
-
- public Builder setNodeReport(SCMNodeReport scmNodeReport) {
- this.nodeReport = scmNodeReport;
- return this;
- }
-
- public Builder setContainerReportState(ReportState crs) {
- this.containerReportState = crs;
- return this;
- }
-
- @VisibleForTesting
- public Builder setRecvTimestamp(long recvTime) {
- this.recvTimestamp = recvTime;
- return this;
- }
-
- public HeartbeatQueueItem build() {
- return new HeartbeatQueueItem(datanodeID, recvTimestamp, nodeReport,
- containerReportState);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
deleted file mode 100644
index 77b3bff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManager.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
-import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
-
-import java.io.Closeable;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A node manager supports a simple interface for managing a datanode.
- * <p/>
- * 1. A datanode registers with the NodeManager.
- * <p/>
- * 2. If the node is allowed to register, we add that to the nodes that we need
- * to keep track of.
- * <p/>
- * 3. A heartbeat is made by the node at a fixed frequency.
- * <p/>
- * 4. A node can be in any of these 4 states: {HEALTHY, STALE, DEAD,
- * DECOMMISSIONED}
- * <p/>
- * HEALTHY - It is a datanode that is regularly heartbeating us.
- *
- * STALE - A datanode for which we have missed few heart beats.
- *
- * DEAD - A datanode that we have not heard from for a while.
- *
- * DECOMMISSIONED - Someone told us to remove this node from the tracking
- * list, by calling removeNode. We will throw away this nodes info soon.
- */
-public interface NodeManager extends StorageContainerNodeProtocol,
- NodeManagerMXBean, Closeable, Runnable {
- /**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws UnregisteredNodeException
- */
- void removeNode(DatanodeID node) throws UnregisteredNodeException;
-
- /**
- * Gets all Live Datanodes that is currently communicating with SCM.
- * @param nodeState - State of the node
- * @return List of Datanodes that are Heartbeating SCM.
- */
- List<DatanodeID> getNodes(NodeState nodeState);
-
- /**
- * Returns the Number of Datanodes that are communicating with SCM.
- * @param nodeState - State of the node
- * @return int -- count
- */
- int getNodeCount(NodeState nodeState);
-
- /**
- * Get all datanodes known to SCM.
- *
- * @return List of DatanodeIDs known to SCM.
- */
- List<DatanodeID> getAllNodes();
-
- /**
- * Chill mode is the period when node manager waits for a minimum
- * configured number of datanodes to report in. This is called chill mode
- * to indicate the period before node manager gets into action.
- *
- * Forcefully exits the chill mode, even if we have not met the minimum
- * criteria of the nodes reporting in.
- */
- void forceExitChillMode();
-
- /**
- * Puts the node manager into manual chill mode.
- */
- void enterChillMode();
-
- /**
- * Brings node manager out of manual chill mode.
- */
- void exitChillMode();
-
- /**
- * Returns the aggregated node stats.
- * @return the aggregated node stats.
- */
- SCMNodeStat getStats();
-
- /**
- * Return a map of node stats.
- * @return a map of individual node stats (live/stale but not dead).
- */
- Map<String, SCMNodeStat> getNodeStats();
-
- /**
- * Return the node stat of the specified datanode.
- * @param datanodeID - datanode ID.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
- */
- SCMNodeMetric getNodeStat(DatanodeID datanodeID);
-
- /**
- * Returns the NodePoolManager associated with the NodeManager.
- * @return NodePoolManager
- */
- NodePoolManager getNodePoolManager();
-
- /**
- * Wait for the heartbeat is processed by NodeManager.
- * @return true if heartbeat has been processed.
- */
- @VisibleForTesting
- boolean waitForHeartbeatProcessed();
-
- /**
- * Returns the node state of a specific node.
- * @param id - DatanodeID
- * @return Healthy/Stale/Dead.
- */
- NodeState getNodeState(DatanodeID id);
-
- /**
- * Add a {@link SCMCommand} to the command queue, which are
- * handled by HB thread asynchronously.
- * @param id
- * @param command
- */
- void addDatanodeCommand(DatanodeID id, SCMCommand command);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
deleted file mode 100644
index 46881c9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodeManagerMXBean.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.util.Map;
-
-/**
- *
- * This is the JMX management interface for node manager information.
- */
-@InterfaceAudience.Private
-public interface NodeManagerMXBean {
- /**
- * Get the minimum number of nodes to get out of chill mode.
- *
- * @return int
- */
- int getMinimumChillModeNodes();
-
- /**
- * Returns a chill mode status string.
- * @return String
- */
- String getChillModeStatus();
-
-
- /**
- * Returns true if node manager is out of chill mode, else false.
- * @return true if out of chill mode, else false
- */
- boolean isOutOfChillMode();
-
- /**
- * Get the number of data nodes that in all states.
- *
- * @return A state to number of nodes that in this state mapping
- */
- Map<String, Integer> getNodeCount();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
deleted file mode 100644
index d3218b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/NodePoolManager.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Interface that defines SCM NodePoolManager.
- */
-public interface NodePoolManager extends Closeable {
-
- /**
- * Add a node to a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- */
- void addNode(String pool, DatanodeID node) throws IOException;
-
- /**
- * Remove a node from a node pool.
- * @param pool - name of the node pool.
- * @param node - data node.
- * @throws SCMException
- */
- void removeNode(String pool, DatanodeID node)
- throws SCMException;
-
- /**
- * Get a list of known node pools.
- * @return a list of known node pool names or an empty list if not node pool
- * is defined.
- */
- List<String> getNodePools();
-
- /**
- * Get all nodes of a node pool given the name of the node pool.
- * @param pool - name of the node pool.
- * @return a list of datanode ids or an empty list if the node pool was not
- * found.
- */
- List<DatanodeID> getNodes(String pool);
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeID - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- */
- String getNodePool(DatanodeID datanodeID) throws SCMException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
deleted file mode 100644
index 129e65d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodeManager.java
+++ /dev/null
@@ -1,883 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.UnregisteredNodeException;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.StorageContainerNodeProtocol;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocol.commands.ReregisterCommand;
-import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
- .ErrorCode;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol
- .proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol
- .proto.StorageContainerDatanodeProtocolProtos.SCMStorageReport;
-import org.apache.hadoop.ozone.scm.StorageContainerManager;
-import org.apache.hadoop.ozone.scm.VersionInfo;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeStat;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.DEAD;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.HEALTHY;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.STALE;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState.UNKNOWN;
-import static org.apache.hadoop.util.Time.monotonicNow;
-
-/**
- * Maintains information about the Datanodes on SCM side.
- * <p>
- * Heartbeats under SCM is very simple compared to HDFS heartbeatManager.
- * <p>
- * Here we maintain 3 maps, and we propagate a node from healthyNodesMap to
- * staleNodesMap to deadNodesMap. This moving of a node from one map to another
- * is controlled by 4 configuration variables. These variables define how many
- * heartbeats must go missing for the node to move from one map to another.
- * <p>
- * Each heartbeat that SCMNodeManager receives is put into heartbeatQueue. The
- * worker thread wakes up and grabs that heartbeat from the queue. The worker
- * thread will lookup the healthynodes map and set the timestamp if the entry
- * is there. if not it will look up stale and deadnodes map.
- * <p>
- * The getNode(byState) functions make copy of node maps and then creates a list
- * based on that. It should be assumed that these get functions always report
- * *stale* information. For example, getting the deadNodeCount followed by
- * getNodes(DEAD) could very well produce totally different count. Also
- * getNodeCount(HEALTHY) + getNodeCount(DEAD) + getNodeCode(STALE), is not
- * guaranteed to add up to the total nodes that we know off. Please treat all
- * get functions in this file as a snap-shot of information that is inconsistent
- * as soon as you read it.
- */
-public class SCMNodeManager
- implements NodeManager, StorageContainerNodeProtocol {
-
- @VisibleForTesting
- static final Logger LOG =
- LoggerFactory.getLogger(SCMNodeManager.class);
-
- /**
- * Key = NodeID, value = timestamp.
- */
- private final ConcurrentHashMap<String, Long> healthyNodes;
- private final ConcurrentHashMap<String, Long> staleNodes;
- private final ConcurrentHashMap<String, Long> deadNodes;
- private final Queue<HeartbeatQueueItem> heartbeatQueue;
- private final ConcurrentHashMap<String, DatanodeID> nodes;
- // Individual live node stats
- private final ConcurrentHashMap<String, SCMNodeStat> nodeStats;
- // Aggregated node stats
- private SCMNodeStat scmStat;
- // TODO: expose nodeStats and scmStat as metrics
- private final AtomicInteger healthyNodeCount;
- private final AtomicInteger staleNodeCount;
- private final AtomicInteger deadNodeCount;
- private final AtomicInteger totalNodes;
- private long staleNodeIntervalMs;
- private final long deadNodeIntervalMs;
- private final long heartbeatCheckerIntervalMs;
- private final long datanodeHBIntervalSeconds;
- private final ScheduledExecutorService executorService;
- private long lastHBcheckStart;
- private long lastHBcheckFinished = 0;
- private long lastHBProcessedCount;
- private int chillModeNodeCount;
- private final int maxHBToProcessPerLoop;
- private final String clusterID;
- private final VersionInfo version;
- /**
- * During start up of SCM, it will enter into chill mode and will be there
- * until number of Datanodes registered reaches {@code chillModeNodeCount}.
- * This flag is for tracking startup chill mode.
- */
- private AtomicBoolean inStartupChillMode;
- /**
- * Administrator can put SCM into chill mode manually.
- * This flag is for tracking manual chill mode.
- */
- private AtomicBoolean inManualChillMode;
- private final CommandQueue commandQueue;
- // Node manager MXBean
- private ObjectName nmInfoBean;
-
- // Node pool manager.
- private final SCMNodePoolManager nodePoolManager;
- private final StorageContainerManager scmManager;
-
- /**
- * Constructs SCM machine Manager.
- */
- public SCMNodeManager(OzoneConfiguration conf, String clusterID,
- StorageContainerManager scmManager) throws IOException {
- heartbeatQueue = new ConcurrentLinkedQueue<>();
- healthyNodes = new ConcurrentHashMap<>();
- deadNodes = new ConcurrentHashMap<>();
- staleNodes = new ConcurrentHashMap<>();
- nodes = new ConcurrentHashMap<>();
- nodeStats = new ConcurrentHashMap();
- scmStat = new SCMNodeStat();
-
- healthyNodeCount = new AtomicInteger(0);
- staleNodeCount = new AtomicInteger(0);
- deadNodeCount = new AtomicInteger(0);
- totalNodes = new AtomicInteger(0);
- this.clusterID = clusterID;
- this.version = VersionInfo.getLatestVersion();
- commandQueue = new CommandQueue();
-
- // TODO: Support this value as a Percentage of known machines.
- chillModeNodeCount = 1;
-
- staleNodeIntervalMs = OzoneClientUtils.getStaleNodeInterval(conf);
- deadNodeIntervalMs = OzoneClientUtils.getDeadNodeInterval(conf);
- heartbeatCheckerIntervalMs =
- OzoneClientUtils.getScmheartbeatCheckerInterval(conf);
- datanodeHBIntervalSeconds = OzoneClientUtils.getScmHeartbeatInterval(conf);
- maxHBToProcessPerLoop = OzoneClientUtils.getMaxHBToProcessPerLoop(conf);
-
- executorService = HadoopExecutors.newScheduledThreadPool(1,
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("SCM Heartbeat Processing Thread - %d").build());
-
- LOG.info("Entering startup chill mode.");
- this.inStartupChillMode = new AtomicBoolean(true);
- this.inManualChillMode = new AtomicBoolean(false);
-
- Preconditions.checkState(heartbeatCheckerIntervalMs > 0);
- executorService.schedule(this, heartbeatCheckerIntervalMs,
- TimeUnit.MILLISECONDS);
-
- registerMXBean();
-
- this.nodePoolManager = new SCMNodePoolManager(conf);
- this.scmManager = scmManager;
- }
-
- private void registerMXBean() {
- this.nmInfoBean = MBeans.register("SCMNodeManager",
- "SCMNodeManagerInfo", this);
- }
-
- private void unregisterMXBean() {
- if(this.nmInfoBean != null) {
- MBeans.unregister(this.nmInfoBean);
- this.nmInfoBean = null;
- }
- }
-
- /**
- * Removes a data node from the management of this Node Manager.
- *
- * @param node - DataNode.
- * @throws UnregisteredNodeException
- */
- @Override
- public void removeNode(DatanodeID node) throws UnregisteredNodeException {
- // TODO : Fix me when adding the SCM CLI.
-
- }
-
- /**
- * Gets all datanodes that are in a certain state. This function works by
- * taking a snapshot of the current collection and then returning the list
- * from that collection. This means that real map might have changed by the
- * time we return this list.
- *
- * @return List of Datanodes that are known to SCM in the requested state.
- */
- @Override
- public List<DatanodeID> getNodes(NodeState nodestate)
- throws IllegalArgumentException {
- Map<String, Long> set;
- switch (nodestate) {
- case HEALTHY:
- synchronized (this) {
- set = Collections.unmodifiableMap(new HashMap<>(healthyNodes));
- }
- break;
- case STALE:
- synchronized (this) {
- set = Collections.unmodifiableMap(new HashMap<>(staleNodes));
- }
- break;
- case DEAD:
- synchronized (this) {
- set = Collections.unmodifiableMap(new HashMap<>(deadNodes));
- }
- break;
- default:
- throw new IllegalArgumentException("Unknown node state requested.");
- }
-
- return set.entrySet().stream().map(entry -> nodes.get(entry.getKey()))
- .collect(Collectors.toList());
- }
-
- /**
- * Returns all datanodes that are known to SCM.
- *
- * @return List of DatanodeIDs
- */
- @Override
- public List<DatanodeID> getAllNodes() {
- Map<String, DatanodeID> set;
- synchronized (this) {
- set = Collections.unmodifiableMap(new HashMap<>(nodes));
- }
- return set.entrySet().stream().map(entry -> nodes.get(entry.getKey()))
- .collect(Collectors.toList());
- }
-
- /**
- * Get the minimum number of nodes to get out of Chill mode.
- *
- * @return int
- */
- @Override
- public int getMinimumChillModeNodes() {
- return chillModeNodeCount;
- }
-
- /**
- * Sets the Minimum chill mode nodes count, used only in testing.
- *
- * @param count - Number of nodes.
- */
- @VisibleForTesting
- public void setMinimumChillModeNodes(int count) {
- chillModeNodeCount = count;
- }
-
- /**
- * Returns chill mode Status string.
- * @return String
- */
- @Override
- public String getChillModeStatus() {
- if (inStartupChillMode.get()) {
- return "Still in chill mode, waiting on nodes to report in." +
- String.format(" %d nodes reported, minimal %d nodes required.",
- totalNodes.get(), getMinimumChillModeNodes());
- }
- if (inManualChillMode.get()) {
- return "Out of startup chill mode, but in manual chill mode." +
- String.format(" %d nodes have reported in.", totalNodes.get());
- }
- return "Out of chill mode." +
- String.format(" %d nodes have reported in.", totalNodes.get());
- }
-
- /**
- * Forcefully exits the chill mode even if we have not met the minimum
- * criteria of exiting the chill mode. This will exit from both startup
- * and manual chill mode.
- */
- @Override
- public void forceExitChillMode() {
- if(inStartupChillMode.get()) {
- LOG.info("Leaving startup chill mode.");
- inStartupChillMode.set(false);
- }
- if(inManualChillMode.get()) {
- LOG.info("Leaving manual chill mode.");
- inManualChillMode.set(false);
- }
- }
-
- /**
- * Puts the node manager into manual chill mode.
- */
- @Override
- public void enterChillMode() {
- LOG.info("Entering manual chill mode.");
- inManualChillMode.set(true);
- }
-
- /**
- * Brings node manager out of manual chill mode.
- */
- @Override
- public void exitChillMode() {
- LOG.info("Leaving manual chill mode.");
- inManualChillMode.set(false);
- }
-
- /**
- * Returns true if node manager is out of chill mode, else false.
- * @return true if out of chill mode, else false
- */
- @Override
- public boolean isOutOfChillMode() {
- return !(inStartupChillMode.get() || inManualChillMode.get());
- }
-
- /**
- * Returns the Number of Datanodes by State they are in.
- *
- * @return int -- count
- */
- @Override
- public int getNodeCount(NodeState nodestate) {
- switch (nodestate) {
- case HEALTHY:
- return healthyNodeCount.get();
- case STALE:
- return staleNodeCount.get();
- case DEAD:
- return deadNodeCount.get();
- case UNKNOWN:
- // This is unknown due to the fact that some nodes can be in
- // transit between the other states. Returning a count for that is not
- // possible. The fact that we have such state is to deal with the fact
- // that this information might not be consistent always.
- return 0;
- default:
- return 0;
- }
- }
-
- /**
- * Used for testing.
- *
- * @return true if the HB check is done.
- */
- @VisibleForTesting
- @Override
- public boolean waitForHeartbeatProcessed() {
- return lastHBcheckFinished != 0;
- }
-
- /**
- * Returns the node state of a specific node.
- *
- * @param id - DatanodeID
- * @return Healthy/Stale/Dead/Unknown.
- */
- @Override
- public NodeState getNodeState(DatanodeID id) {
- // There is a subtle race condition here, hence we also support
- // the NODEState.UNKNOWN. It is possible that just before we check the
- // healthyNodes, we have removed the node from the healthy list but stil
- // not added it to Stale Nodes list.
- // We can fix that by adding the node to stale list before we remove, but
- // then the node is in 2 states to avoid this race condition. Instead we
- // just deal with the possibilty of getting a state called unknown.
-
- if(healthyNodes.containsKey(id.getDatanodeUuid())) {
- return HEALTHY;
- }
-
- if(staleNodes.containsKey(id.getDatanodeUuid())) {
- return STALE;
- }
-
- if(deadNodes.containsKey(id.getDatanodeUuid())) {
- return DEAD;
- }
-
- return UNKNOWN;
- }
-
- /**
- * This is the real worker thread that processes the HB queue. We do the
- * following things in this thread.
- * <p>
- * Process the Heartbeats that are in the HB Queue. Move Stale or Dead node to
- * healthy if we got a heartbeat from them. Move Stales Node to dead node
- * table if it is needed. Move healthy nodes to stale nodes if it is needed.
- * <p>
- * if it is a new node, we call register node and add it to the list of nodes.
- * This will be replaced when we support registration of a node in SCM.
- *
- * @see Thread#run()
- */
- @Override
- public void run() {
- lastHBcheckStart = monotonicNow();
- lastHBProcessedCount = 0;
-
- // Process the whole queue.
- while (!heartbeatQueue.isEmpty() &&
- (lastHBProcessedCount < maxHBToProcessPerLoop)) {
- HeartbeatQueueItem hbItem = heartbeatQueue.poll();
- synchronized (this) {
- handleHeartbeat(hbItem);
- }
- // we are shutting down or something give up processing the rest of
- // HBs. This will terminate the HB processing thread.
- if (Thread.currentThread().isInterrupted()) {
- LOG.info("Current Thread is isInterrupted, shutting down HB " +
- "processing thread for Node Manager.");
- return;
- }
- }
-
- if (lastHBProcessedCount >= maxHBToProcessPerLoop) {
- LOG.error("SCM is being flooded by heartbeats. Not able to keep up with" +
- " the heartbeat counts. Processed {} heartbeats. Breaking out of" +
- " loop. Leaving rest to be processed later. ", lastHBProcessedCount);
- }
-
- // Iterate over the Stale nodes and decide if we need to move any node to
- // dead State.
- long currentTime = monotonicNow();
- for (Map.Entry<String, Long> entry : staleNodes.entrySet()) {
- if (currentTime - entry.getValue() > deadNodeIntervalMs) {
- synchronized (this) {
- moveStaleNodeToDead(entry);
- }
- }
- }
-
- // Iterate over the healthy nodes and decide if we need to move any node to
- // Stale State.
- currentTime = monotonicNow();
- for (Map.Entry<String, Long> entry : healthyNodes.entrySet()) {
- if (currentTime - entry.getValue() > staleNodeIntervalMs) {
- synchronized (this) {
- moveHealthyNodeToStale(entry);
- }
- }
- }
- lastHBcheckFinished = monotonicNow();
-
- monitorHBProcessingTime();
-
- // we purposefully make this non-deterministic. Instead of using a
- // scheduleAtFixedFrequency we will just go to sleep
- // and wake up at the next rendezvous point, which is currentTime +
- // heartbeatCheckerIntervalMs. This leads to the issue that we are now
- // heart beating not at a fixed cadence, but clock tick + time taken to
- // work.
- //
- // This time taken to work can skew the heartbeat processor thread.
- // The reason why we don't care is because of the following reasons.
- //
- // 1. checkerInterval is general many magnitudes faster than datanode HB
- // frequency.
- //
- // 2. if we have too much nodes, the SCM would be doing only HB
- // processing, this could lead to SCM's CPU starvation. With this
- // approach we always guarantee that HB thread sleeps for a little while.
- //
- // 3. It is possible that we will never finish processing the HB's in the
- // thread. But that means we have a mis-configured system. We will warn
- // the users by logging that information.
- //
- // 4. And the most important reason, heartbeats are not blocked even if
- // this thread does not run, they will go into the processing queue.
-
- if (!Thread.currentThread().isInterrupted() &&
- !executorService.isShutdown()) {
- executorService.schedule(this, heartbeatCheckerIntervalMs, TimeUnit
- .MILLISECONDS);
- } else {
- LOG.info("Current Thread is interrupted, shutting down HB processing " +
- "thread for Node Manager.");
- }
- }
-
- /**
- * If we have taken too much time for HB processing, log that information.
- */
- private void monitorHBProcessingTime() {
- if (TimeUnit.MILLISECONDS.toSeconds(lastHBcheckFinished -
- lastHBcheckStart) > datanodeHBIntervalSeconds) {
- LOG.error("Total time spend processing datanode HB's is greater than " +
- "configured values for datanode heartbeats. Please adjust the" +
- " heartbeat configs. Time Spend on HB processing: {} seconds " +
- "Datanode heartbeat Interval: {} seconds , heartbeats " +
- "processed: {}",
- TimeUnit.MILLISECONDS
- .toSeconds(lastHBcheckFinished - lastHBcheckStart),
- datanodeHBIntervalSeconds, lastHBProcessedCount);
- }
- }
-
- /**
- * Moves a Healthy node to a Stale node state.
- *
- * @param entry - Map Entry
- */
- private void moveHealthyNodeToStale(Map.Entry<String, Long> entry) {
- LOG.trace("Moving healthy node to stale: {}", entry.getKey());
- healthyNodes.remove(entry.getKey());
- healthyNodeCount.decrementAndGet();
- staleNodes.put(entry.getKey(), entry.getValue());
- staleNodeCount.incrementAndGet();
-
- if (scmManager != null) {
- // remove stale node's container report
- scmManager.removeContainerReport(entry.getKey());
- }
- }
-
- /**
- * Moves a Stale node to a dead node state.
- *
- * @param entry - Map Entry
- */
- private void moveStaleNodeToDead(Map.Entry<String, Long> entry) {
- LOG.trace("Moving stale node to dead: {}", entry.getKey());
- staleNodes.remove(entry.getKey());
- staleNodeCount.decrementAndGet();
- deadNodes.put(entry.getKey(), entry.getValue());
- deadNodeCount.incrementAndGet();
-
- // Update SCM node stats
- SCMNodeStat deadNodeStat = nodeStats.get(entry.getKey());
- scmStat.subtract(deadNodeStat);
- nodeStats.remove(entry.getKey());
- }
-
- /**
- * Handles a single heartbeat from a datanode.
- *
- * @param hbItem - heartbeat item from a datanode.
- */
- private void handleHeartbeat(HeartbeatQueueItem hbItem) {
- lastHBProcessedCount++;
-
- DatanodeID datanodeID = hbItem.getDatanodeID();
- String datanodeUuid = datanodeID.getDatanodeUuid();
- SCMNodeReport nodeReport = hbItem.getNodeReport();
- long recvTimestamp = hbItem.getRecvTimestamp();
- long processTimestamp = Time.monotonicNow();
- if (LOG.isTraceEnabled()) {
- //TODO: add average queue time of heartbeat request as metrics
- LOG.trace("Processing Heartbeat from datanode {}: queueing time {}",
- datanodeUuid, processTimestamp - recvTimestamp);
- }
-
- // If this node is already in the list of known and healthy nodes
- // just set the last timestamp and return.
- if (healthyNodes.containsKey(datanodeUuid)) {
- healthyNodes.put(datanodeUuid, processTimestamp);
- updateNodeStat(datanodeUuid, nodeReport);
- updateCommandQueue(datanodeID,
- hbItem.getContainerReportState().getState());
- return;
- }
-
- // A stale node has heartbeat us we need to remove the node from stale
- // list and move to healthy list.
- if (staleNodes.containsKey(datanodeUuid)) {
- staleNodes.remove(datanodeUuid);
- healthyNodes.put(datanodeUuid, processTimestamp);
- healthyNodeCount.incrementAndGet();
- staleNodeCount.decrementAndGet();
- updateNodeStat(datanodeUuid, nodeReport);
- updateCommandQueue(datanodeID,
- hbItem.getContainerReportState().getState());
- return;
- }
-
- // A dead node has heartbeat us, we need to remove that node from dead
- // node list and move it to the healthy list.
- if (deadNodes.containsKey(datanodeUuid)) {
- deadNodes.remove(datanodeUuid);
- healthyNodes.put(datanodeUuid, processTimestamp);
- deadNodeCount.decrementAndGet();
- healthyNodeCount.incrementAndGet();
- updateNodeStat(datanodeUuid, nodeReport);
- updateCommandQueue(datanodeID,
- hbItem.getContainerReportState().getState());
- return;
- }
-
- LOG.warn("SCM receive heartbeat from unregistered datanode {}",
- datanodeUuid);
- this.commandQueue.addCommand(hbItem.getDatanodeID(),
- new ReregisterCommand());
- }
-
- private void updateNodeStat(String datanodeUuid, SCMNodeReport nodeReport) {
- SCMNodeStat stat = nodeStats.get(datanodeUuid);
- if (stat == null) {
- LOG.debug("SCM updateNodeStat based on heartbeat from previous" +
- "dead datanode {}", datanodeUuid);
- stat = new SCMNodeStat();
- }
-
- if (nodeReport != null && nodeReport.getStorageReportCount() > 0) {
- long totalCapacity = 0;
- long totalRemaining = 0;
- long totalScmUsed = 0;
- List<SCMStorageReport> storageReports = nodeReport.getStorageReportList();
- for (SCMStorageReport report : storageReports) {
- totalCapacity += report.getCapacity();
- totalRemaining += report.getRemaining();
- totalScmUsed+= report.getScmUsed();
- }
- scmStat.subtract(stat);
- stat.set(totalCapacity, totalScmUsed, totalRemaining);
- nodeStats.put(datanodeUuid, stat);
- scmStat.add(stat);
- }
- }
-
- private void updateCommandQueue(DatanodeID datanodeID,
- ReportState.states containerReportState) {
- if (containerReportState != null) {
- switch (containerReportState) {
- case completeContinerReport:
- commandQueue.addCommand(datanodeID,
- SendContainerCommand.newBuilder().build());
- return;
- case deltaContainerReport:
- case noContainerReports:
- default:
- // do nothing
- }
- }
- }
-
- /**
- * Closes this stream and releases any system resources associated with it. If
- * the stream is already closed then invoking this method has no effect.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- unregisterMXBean();
- nodePoolManager.close();
- executorService.shutdown();
- try {
- if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
- executorService.shutdownNow();
- }
-
- if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
- LOG.error("Unable to shutdown NodeManager properly.");
- }
- } catch (InterruptedException e) {
- executorService.shutdownNow();
- Thread.currentThread().interrupt();
- }
- }
-
- @VisibleForTesting
- long getLastHBProcessedCount() {
- return lastHBProcessedCount;
- }
-
- /**
- * Gets the version info from SCM.
- *
- * @param versionRequest - version Request.
- * @return - returns SCM version info and other required information needed by
- * datanode.
- */
- @Override
- public VersionResponse getVersion(SCMVersionRequestProto versionRequest) {
- return VersionResponse.newBuilder()
- .setVersion(this.version.getVersion())
- .build();
- }
-
- /**
- * Register the node if the node finds that it is not registered with any
- * SCM.
- *
- * @param datanodeID - Send datanodeID with Node info. This function
- * generates and assigns new datanode ID for the datanode.
- * This allows SCM to be run independent of Namenode if
- * required.
- *
- * @return SCMHeartbeatResponseProto
- */
- @Override
- public SCMCommand register(DatanodeID datanodeID) {
-
- SCMCommand responseCommand = verifyDatanodeUUID(datanodeID);
- if (responseCommand != null) {
- return responseCommand;
- }
-
- nodes.put(datanodeID.getDatanodeUuid(), datanodeID);
- totalNodes.incrementAndGet();
- healthyNodes.put(datanodeID.getDatanodeUuid(), monotonicNow());
- healthyNodeCount.incrementAndGet();
- nodeStats.put(datanodeID.getDatanodeUuid(), new SCMNodeStat());
-
- if(inStartupChillMode.get() &&
- totalNodes.get() >= getMinimumChillModeNodes()) {
- inStartupChillMode.getAndSet(false);
- LOG.info("Leaving startup chill mode.");
- }
-
- // TODO: define node pool policy for non-default node pool.
- // For now, all nodes are added to the "DefaultNodePool" upon registration
- // if it has not been added to any node pool yet.
- try {
- if (nodePoolManager.getNodePool(datanodeID) == null) {
- nodePoolManager.addNode(SCMNodePoolManager.DEFAULT_NODEPOOL,
- datanodeID);
- }
- } catch (IOException e) {
- // TODO: make sure registration failure is handled correctly.
- return RegisteredCommand.newBuilder()
- .setErrorCode(ErrorCode.errorNodeNotPermitted)
- .build();
- }
- LOG.info("Data node with ID: {} Registered.",
- datanodeID.getDatanodeUuid());
- return RegisteredCommand.newBuilder()
- .setErrorCode(ErrorCode.success)
- .setDatanodeUUID(datanodeID.getDatanodeUuid())
- .setClusterID(this.clusterID)
- .build();
- }
-
- /**
- * Verifies the datanode does not have a valid UUID already.
- *
- * @param datanodeID - Datanode UUID.
- * @return SCMCommand
- */
- private SCMCommand verifyDatanodeUUID(DatanodeID datanodeID) {
- if (datanodeID.getDatanodeUuid() != null &&
- nodes.containsKey(datanodeID.getDatanodeUuid())) {
- LOG.trace("Datanode is already registered. Datanode: {}",
- datanodeID.toString());
- return RegisteredCommand.newBuilder()
- .setErrorCode(ErrorCode.success)
- .setClusterID(this.clusterID)
- .setDatanodeUUID(datanodeID.getDatanodeUuid())
- .build();
- }
- return null;
- }
-
- /**
- * Send heartbeat to indicate the datanode is alive and doing well.
- *
- * @param datanodeID - Datanode ID.
- * @param nodeReport - node report.
- * @param containerReportState - container report state.
- * @return SCMheartbeat response.
- * @throws IOException
- */
- @Override
- public List<SCMCommand> sendHeartbeat(DatanodeID datanodeID,
- SCMNodeReport nodeReport, ReportState containerReportState) {
-
- // Checking for NULL to make sure that we don't get
- // an exception from ConcurrentList.
- // This could be a problem in tests, if this function is invoked via
- // protobuf, transport layer will guarantee that this is not null.
- if (datanodeID != null) {
- heartbeatQueue.add(
- new HeartbeatQueueItem.Builder()
- .setDatanodeID(datanodeID)
- .setNodeReport(nodeReport)
- .setContainerReportState(containerReportState)
- .build());
- } else {
- LOG.error("Datanode ID in heartbeat is null");
- }
-
- return commandQueue.getCommand(datanodeID);
- }
-
- /**
- * Returns the aggregated node stats.
- * @return the aggregated node stats.
- */
- @Override
- public SCMNodeStat getStats() {
- return new SCMNodeStat(this.scmStat);
- }
-
- /**
- * Return a map of node stats.
- * @return a map of individual node stats (live/stale but not dead).
- */
- @Override
- public Map<String, SCMNodeStat> getNodeStats() {
- return Collections.unmodifiableMap(nodeStats);
- }
-
- /**
- * Return the node stat of the specified datanode.
- * @param datanodeID - datanode ID.
- * @return node stat if it is live/stale, null if it is dead or does't exist.
- */
- @Override
- public SCMNodeMetric getNodeStat(DatanodeID datanodeID) {
- return new SCMNodeMetric(nodeStats.get(datanodeID.getDatanodeUuid()));
- }
-
- @Override
- public NodePoolManager getNodePoolManager() {
- return nodePoolManager;
- }
-
- @Override
- public Map<String, Integer> getNodeCount() {
- Map<String, Integer> nodeCountMap = new HashMap<String, Integer>();
- for(NodeState state : NodeState.values()) {
- nodeCountMap.put(state.toString(), getNodeCount(state));
- }
- return nodeCountMap;
- }
-
- @Override
- public void addDatanodeCommand(DatanodeID id, SCMCommand command) {
- this.commandQueue.addCommand(id, command);
- }
-
- @VisibleForTesting
- public void setStaleNodeIntervalMs(long interval) {
- this.staleNodeIntervalMs = interval;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
deleted file mode 100644
index 55bf1c4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/SCMNodePoolManager.java
+++ /dev/null
@@ -1,266 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.node;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-import static org.apache.hadoop.ozone.scm
- .exceptions.SCMException.ResultCodes.FAILED_TO_LOAD_NODEPOOL;
-import static org.apache.hadoop.ozone.scm
- .exceptions.SCMException.ResultCodes.FAILED_TO_FIND_NODE_IN_POOL;
-import static org.apache.hadoop.scm
- .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm
- .ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-
-/**
- * SCM node pool manager that manges node pools.
- */
-public final class SCMNodePoolManager implements NodePoolManager {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMNodePoolManager.class);
- private static final List<DatanodeID> EMPTY_NODE_LIST = new ArrayList<>();
- private static final List<String> EMPTY_NODEPOOL_LIST = new ArrayList<>();
- public static final String DEFAULT_NODEPOOL = "DefaultNodePool";
-
- // DB that saves the node to node pool mapping.
- private MetadataStore nodePoolStore;
-
- // In-memory node pool to nodes mapping
- private HashMap<String, Set<DatanodeID>> nodePools;
-
- // Read-write lock for nodepool operations
- private ReadWriteLock lock;
-
- /**
- * Construct SCMNodePoolManager class that manages node to node pool mapping.
- * @param conf - configuration.
- * @throws IOException
- */
- public SCMNodePoolManager(final OzoneConfiguration conf)
- throws IOException {
- final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
- File nodePoolDBPath = new File(scmMetaDataDir, NODEPOOL_DB);
- nodePoolStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(nodePoolDBPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
- nodePools = new HashMap<>();
- lock = new ReentrantReadWriteLock();
- init();
- }
-
- /**
- * Initialize the in-memory store based on persist store from level db.
- * No lock is needed as init() is only invoked by constructor.
- * @throws SCMException
- */
- private void init() throws SCMException {
- try {
- nodePoolStore.iterate(null, (key, value) -> {
- try {
- DatanodeID nodeId = DatanodeID.getFromProtoBuf(
- HdfsProtos.DatanodeIDProto.PARSER.parseFrom(key));
- String poolName = DFSUtil.bytes2String(value);
-
- Set<DatanodeID> nodePool = null;
- if (nodePools.containsKey(poolName)) {
- nodePool = nodePools.get(poolName);
- } else {
- nodePool = new HashSet<>();
- nodePools.put(poolName, nodePool);
- }
- nodePool.add(nodeId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding node: {} to node pool: {}",
- nodeId, poolName);
- }
- } catch (IOException e) {
- LOG.warn("Can't add a datanode to node pool, continue next...");
- }
- return true;
- });
- } catch (IOException e) {
- LOG.error("Loading node pool error " + e);
- throw new SCMException("Failed to load node pool",
- FAILED_TO_LOAD_NODEPOOL);
- }
- }
-
- /**
- * Add a datanode to a node pool.
- * @param pool - name of the node pool.
- * @param node - name of the datanode.
- */
- @Override
- public void addNode(final String pool, final DatanodeID node)
- throws IOException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // add to the persistent store
- nodePoolStore.put(node.getProtoBufMessage().toByteArray(),
- DFSUtil.string2Bytes(pool));
-
- // add to the in-memory store
- Set<DatanodeID> nodePool = null;
- if (nodePools.containsKey(pool)) {
- nodePool = nodePools.get(pool);
- } else {
- nodePool = new HashSet<DatanodeID>();
- nodePools.put(pool, nodePool);
- }
- nodePool.add(node);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Remove a datanode from a node pool.
- * @param pool - name of the node pool.
- * @param node - datanode id.
- * @throws SCMException
- */
- @Override
- public void removeNode(final String pool, final DatanodeID node)
- throws SCMException {
- Preconditions.checkNotNull(pool, "pool name is null");
- Preconditions.checkNotNull(node, "node is null");
- lock.writeLock().lock();
- try {
- // Remove from the persistent store
- byte[] kName = node.getProtoBufMessage().toByteArray();
- byte[] kData = nodePoolStore.get(kName);
- if (kData == null) {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in DB.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL);
- }
- nodePoolStore.delete(kName);
-
- // Remove from the in-memory store
- if (nodePools.containsKey(pool)) {
- Set<DatanodeID> nodePool = nodePools.get(pool);
- nodePool.remove(node);
- } else {
- throw new SCMException(String.format("Unable to find node %s from" +
- " pool %s in MAP.", DFSUtil.bytes2String(kName), pool),
- FAILED_TO_FIND_NODE_IN_POOL); }
- } catch (IOException e) {
- throw new SCMException("Failed to remove node " + node.toString()
- + " from node pool " + pool, e,
- SCMException.ResultCodes.IO_EXCEPTION);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Get all the node pools.
- * @return all the node pools.
- */
- @Override
- public List<String> getNodePools() {
- lock.readLock().lock();
- try {
- if (!nodePools.isEmpty()) {
- return nodePools.keySet().stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODEPOOL_LIST;
- }
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Get all datanodes of a specific node pool.
- * @param pool - name of the node pool.
- * @return all datanodes of the specified node pool.
- */
- @Override
- public List<DatanodeID> getNodes(final String pool) {
- Preconditions.checkNotNull(pool, "pool name is null");
- if (nodePools.containsKey(pool)) {
- return nodePools.get(pool).stream().collect(Collectors.toList());
- } else {
- return EMPTY_NODE_LIST;
- }
- }
-
- /**
- * Get the node pool name if the node has been added to a node pool.
- * @param datanodeID - datanode ID.
- * @return node pool name if it has been assigned.
- * null if the node has not been assigned to any node pool yet.
- * TODO: Put this in a in-memory map if performance is an issue.
- */
- @Override
- public String getNodePool(final DatanodeID datanodeID) throws SCMException {
- Preconditions.checkNotNull(datanodeID, "node is null");
- try {
- byte[] result = nodePoolStore.get(
- datanodeID.getProtoBufMessage().toByteArray());
- return result == null ? null : DFSUtil.bytes2String(result);
- } catch (IOException e) {
- throw new SCMException("Failed to get node pool for node "
- + datanodeID.toString(), e,
- SCMException.ResultCodes.IO_EXCEPTION);
- }
- }
-
- /**
- * Close node pool level db store.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- nodePoolStore.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
deleted file mode 100644
index f1efe79..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/node/package-info.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.node;
-
-/**
- * The node package deals with node management.
- * <p/>
- * The node manager takes care of node registrations, removal of node and
- * handling of heartbeats.
- * <p/>
- * The node manager maintains statistics that gets send as part of
- * heartbeats.
- * <p/>
- * The container manager polls the node manager to learn the state of
- * datanodes that it is interested in.
- * <p/>
- */
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
deleted file mode 100644
index 1664873..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/d3-3.5.17.min.js
+++ /dev/null
@@ -1,5 +0,0 @@
-!function(){function n(n){return n&&(n.ownerDocument||n.document||n).documentElement}function t(n){return n&&(n.ownerDocument&&n.ownerDocument.defaultView||n.document&&n||n.defaultView)}function e(n,t){return t>n?-1:n>t?1:n>=t?0:NaN}function r(n){return null===n?NaN:+n}function i(n){return!isNaN(n)}function u(n){return{left:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)<0?r=u+1:i=u}return r},right:function(t,e,r,i){for(arguments.length<3&&(r=0),arguments.length<4&&(i=t.length);i>r;){var u=r+i>>>1;n(t[u],e)>0?i=u:r=u+1}return r}}}function o(n){return n.length}function a(n){for(var t=1;n*t%1;)t*=10;return t}function l(n,t){for(var e in t)Object.defineProperty(n.prototype,e,{value:t[e],enumerable:!1})}function c(){this._=Object.create(null)}function f(n){return(n+="")===bo||n[0]===_o?_o+n:n}function s(n){return(n+="")[0]===_o?n.slice(1):n}function h(n){return f(n)in this._}function p(n){return(n=f(n))in this._&&delete this
._[n]}function g(){var n=[];for(var t in this._)n.push(s(t));return n}function v(){var n=0;for(var t in this._)++n;return n}function d(){for(var n in this._)return!1;return!0}function y(){this._=Object.create(null)}function m(n){return n}function M(n,t,e){return function(){var r=e.apply(t,arguments);return r===t?n:r}}function x(n,t){if(t in n)return t;t=t.charAt(0).toUpperCase()+t.slice(1);for(var e=0,r=wo.length;r>e;++e){var i=wo[e]+t;if(i in n)return i}}function b(){}function _(){}function w(n){function t(){for(var t,r=e,i=-1,u=r.length;++i<u;)(t=r[i].on)&&t.apply(this,arguments);return n}var e=[],r=new c;return t.on=function(t,i){var u,o=r.get(t);return arguments.length<2?o&&o.on:(o&&(o.on=null,e=e.slice(0,u=e.indexOf(o)).concat(e.slice(u+1)),r.remove(t)),i&&e.push(r.set(t,{on:i})),n)},t}function S(){ao.event.preventDefault()}function k(){for(var n,t=ao.event;n=t.sourceEvent;)t=n;return t}function N(n){for(var t=new _,e=0,r=arguments.length;++e<r;)t[arguments[e]]=w(t);return t.of
=function(e,r){return function(i){try{var u=i.sourceEvent=ao.event;i.target=n,ao.event=i,t[i.type].apply(e,r)}finally{ao.event=u}}},t}function E(n){return ko(n,Co),n}function A(n){return"function"==typeof n?n:function(){return No(n,this)}}function C(n){return"function"==typeof n?n:function(){return Eo(n,this)}}function z(n,t){function e(){this.removeAttribute(n)}function r(){this.removeAttributeNS(n.space,n.local)}function i(){this.setAttribute(n,t)}function u(){this.setAttributeNS(n.space,n.local,t)}function o(){var e=t.apply(this,arguments);null==e?this.removeAttribute(n):this.setAttribute(n,e)}function a(){var e=t.apply(this,arguments);null==e?this.removeAttributeNS(n.space,n.local):this.setAttributeNS(n.space,n.local,e)}return n=ao.ns.qualify(n),null==t?n.local?r:e:"function"==typeof t?n.local?a:o:n.local?u:i}function L(n){return n.trim().replace(/\s+/g," ")}function q(n){return new RegExp("(?:^|\\s+)"+ao.requote(n)+"(?:\\s+|$)","g")}function T(n){return(n+"").trim().split(/^|\s
+/)}function R(n,t){function e(){for(var e=-1;++e<i;)n[e](this,t)}function r(){for(var e=-1,r=t.apply(this,arguments);++e<i;)n[e](this,r)}n=T(n).map(D);var i=n.length;return"function"==typeof t?r:e}function D(n){var t=q(n);return function(e,r){if(i=e.classList)return r?i.add(n):i.remove(n);var i=e.getAttribute("class")||"";r?(t.lastIndex=0,t.test(i)||e.setAttribute("class",L(i+" "+n))):e.setAttribute("class",L(i.replace(t," ")))}}function P(n,t,e){function r(){this.style.removeProperty(n)}function i(){this.style.setProperty(n,t,e)}function u(){var r=t.apply(this,arguments);null==r?this.style.removeProperty(n):this.style.setProperty(n,r,e)}return null==t?r:"function"==typeof t?u:i}function U(n,t){function e(){delete this[n]}function r(){this[n]=t}function i(){var e=t.apply(this,arguments);null==e?delete this[n]:this[n]=e}return null==t?e:"function"==typeof t?i:r}function j(n){function t(){var t=this.ownerDocument,e=this.namespaceURI;return e===zo&&t.documentElement.namespaceURI===zo?
t.createElement(n):t.createElementNS(e,n)}function e(){return this.ownerDocument.createElementNS(n.space,n.local)}return"function"==typeof n?n:(n=ao.ns.qualify(n)).local?e:t}function F(){var n=this.parentNode;n&&n.removeChild(this)}function H(n){return{__data__:n}}function O(n){return function(){return Ao(this,n)}}function I(n){return arguments.length||(n=e),function(t,e){return t&&e?n(t.__data__,e.__data__):!t-!e}}function Y(n,t){for(var e=0,r=n.length;r>e;e++)for(var i,u=n[e],o=0,a=u.length;a>o;o++)(i=u[o])&&t(i,o,e);return n}function Z(n){return ko(n,qo),n}function V(n){var t,e;return function(r,i,u){var o,a=n[u].update,l=a.length;for(u!=e&&(e=u,t=0),i>=t&&(t=i+1);!(o=a[t])&&++t<l;);return o}}function X(n,t,e){function r(){var t=this[o];t&&(this.removeEventListener(n,t,t.$),delete this[o])}function i(){var i=l(t,co(arguments));r.call(this),this.addEventListener(n,this[o]=i,i.$=e),i._=t}function u(){var t,e=new RegExp("^__on([^.]+)"+ao.requote(n)+"$");for(var r in this)if(t=r.matc
h(e)){var i=this[r];this.removeEventListener(t[1],i,i.$),delete this[r]}}var o="__on"+n,a=n.indexOf("."),l=$;a>0&&(n=n.slice(0,a));var c=To.get(n);return c&&(n=c,l=B),a?t?i:r:t?b:u}function $(n,t){return function(e){var r=ao.event;ao.event=e,t[0]=this.__data__;try{n.apply(this,t)}finally{ao.event=r}}}function B(n,t){var e=$(n,t);return function(n){var t=this,r=n.relatedTarget;r&&(r===t||8&r.compareDocumentPosition(t))||e.call(t,n)}}function W(e){var r=".dragsuppress-"+ ++Do,i="click"+r,u=ao.select(t(e)).on("touchmove"+r,S).on("dragstart"+r,S).on("selectstart"+r,S);if(null==Ro&&(Ro="onselectstart"in e?!1:x(e.style,"userSelect")),Ro){var o=n(e).style,a=o[Ro];o[Ro]="none"}return function(n){if(u.on(r,null),Ro&&(o[Ro]=a),n){var t=function(){u.on(i,null)};u.on(i,function(){S(),t()},!0),setTimeout(t,0)}}}function J(n,e){e.changedTouches&&(e=e.changedTouches[0]);var r=n.ownerSVGElement||n;if(r.createSVGPoint){var i=r.createSVGPoint();if(0>Po){var u=t(n);if(u.scrollX||u.scrollY){r=ao.select
("body").append("svg").style({position:"absolute",top:0,left:0,margin:0,padding:0,border:"none"},"important");var o=r[0][0].getScreenCTM();Po=!(o.f||o.e),r.remove()}}return Po?(i.x=e.pageX,i.y=e.pageY):(i.x=e.clientX,i.y=e.clientY),i=i.matrixTransform(n.getScreenCTM().inverse()),[i.x,i.y]}var a=n.getBoundingClientRect();return[e.clientX-a.left-n.clientLeft,e.clientY-a.top-n.clientTop]}function G(){return ao.event.changedTouches[0].identifier}function K(n){return n>0?1:0>n?-1:0}function Q(n,t,e){return(t[0]-n[0])*(e[1]-n[1])-(t[1]-n[1])*(e[0]-n[0])}function nn(n){return n>1?0:-1>n?Fo:Math.acos(n)}function tn(n){return n>1?Io:-1>n?-Io:Math.asin(n)}function en(n){return((n=Math.exp(n))-1/n)/2}function rn(n){return((n=Math.exp(n))+1/n)/2}function un(n){return((n=Math.exp(2*n))-1)/(n+1)}function on(n){return(n=Math.sin(n/2))*n}function an(){}function ln(n,t,e){return this instanceof ln?(this.h=+n,this.s=+t,void(this.l=+e)):arguments.length<2?n instanceof ln?new ln(n.h,n.s,n.l):_n(""+n,wn
,ln):new ln(n,t,e)}function cn(n,t,e){function r(n){return n>360?n-=360:0>n&&(n+=360),60>n?u+(o-u)*n/60:180>n?o:240>n?u+(o-u)*(240-n)/60:u}function i(n){return Math.round(255*r(n))}var u,o;return n=isNaN(n)?0:(n%=360)<0?n+360:n,t=isNaN(t)?0:0>t?0:t>1?1:t,e=0>e?0:e>1?1:e,o=.5>=e?e*(1+t):e+t-e*t,u=2*e-o,new mn(i(n+120),i(n),i(n-120))}function fn(n,t,e){return this instanceof fn?(this.h=+n,this.c=+t,void(this.l=+e)):arguments.length<2?n instanceof fn?new fn(n.h,n.c,n.l):n instanceof hn?gn(n.l,n.a,n.b):gn((n=Sn((n=ao.rgb(n)).r,n.g,n.b)).l,n.a,n.b):new fn(n,t,e)}function sn(n,t,e){return isNaN(n)&&(n=0),isNaN(t)&&(t=0),new hn(e,Math.cos(n*=Yo)*t,Math.sin(n)*t)}function hn(n,t,e){return this instanceof hn?(this.l=+n,this.a=+t,void(this.b=+e)):arguments.length<2?n instanceof hn?new hn(n.l,n.a,n.b):n instanceof fn?sn(n.h,n.c,n.l):Sn((n=mn(n)).r,n.g,n.b):new hn(n,t,e)}function pn(n,t,e){var r=(n+16)/116,i=r+t/500,u=r-e/200;return i=vn(i)*na,r=vn(r)*ta,u=vn(u)*ea,new mn(yn(3.2404542*i-1.53713
85*r-.4985314*u),yn(-.969266*i+1.8760108*r+.041556*u),yn(.0556434*i-.2040259*r+1.0572252*u))}function gn(n,t,e){return n>0?new fn(Math.atan2(e,t)*Zo,Math.sqrt(t*t+e*e),n):new fn(NaN,NaN,n)}function vn(n){return n>.206893034?n*n*n:(n-4/29)/7.787037}function dn(n){return n>.008856?Math.pow(n,1/3):7.787037*n+4/29}function yn(n){return Math.round(255*(.00304>=n?12.92*n:1.055*Math.pow(n,1/2.4)-.055))}function mn(n,t,e){return this instanceof mn?(this.r=~~n,this.g=~~t,void(this.b=~~e)):arguments.length<2?n instanceof mn?new mn(n.r,n.g,n.b):_n(""+n,mn,cn):new mn(n,t,e)}function Mn(n){return new mn(n>>16,n>>8&255,255&n)}function xn(n){return Mn(n)+""}function bn(n){return 16>n?"0"+Math.max(0,n).toString(16):Math.min(255,n).toString(16)}function _n(n,t,e){var r,i,u,o=0,a=0,l=0;if(r=/([a-z]+)\((.*)\)/.exec(n=n.toLowerCase()))switch(i=r[2].split(","),r[1]){case"hsl":return e(parseFloat(i[0]),parseFloat(i[1])/100,parseFloat(i[2])/100);case"rgb":return t(Nn(i[0]),Nn(i[1]),Nn(i[2]))}return(u=ua.g
et(n))?t(u.r,u.g,u.b):(null==n||"#"!==n.charAt(0)||isNaN(u=parseInt(n.slice(1),16))||(4===n.length?(o=(3840&u)>>4,o=o>>4|o,a=240&u,a=a>>4|a,l=15&u,l=l<<4|l):7===n.length&&(o=(16711680&u)>>16,a=(65280&u)>>8,l=255&u)),t(o,a,l))}function wn(n,t,e){var r,i,u=Math.min(n/=255,t/=255,e/=255),o=Math.max(n,t,e),a=o-u,l=(o+u)/2;return a?(i=.5>l?a/(o+u):a/(2-o-u),r=n==o?(t-e)/a+(e>t?6:0):t==o?(e-n)/a+2:(n-t)/a+4,r*=60):(r=NaN,i=l>0&&1>l?0:r),new ln(r,i,l)}function Sn(n,t,e){n=kn(n),t=kn(t),e=kn(e);var r=dn((.4124564*n+.3575761*t+.1804375*e)/na),i=dn((.2126729*n+.7151522*t+.072175*e)/ta),u=dn((.0193339*n+.119192*t+.9503041*e)/ea);return hn(116*i-16,500*(r-i),200*(i-u))}function kn(n){return(n/=255)<=.04045?n/12.92:Math.pow((n+.055)/1.055,2.4)}function Nn(n){var t=parseFloat(n);return"%"===n.charAt(n.length-1)?Math.round(2.55*t):t}function En(n){return"function"==typeof n?n:function(){return n}}function An(n){return function(t,e,r){return 2===arguments.length&&"function"==typeof e&&(r=e,e=null),
Cn(t,e,n,r)}}function Cn(n,t,e,r){function i(){var n,t=l.status;if(!t&&Ln(l)||t>=200&&300>t||304===t){try{n=e.call(u,l)}catch(r){return void o.error.call(u,r)}o.load.call(u,n)}else o.error.call(u,l)}var u={},o=ao.dispatch("beforesend","progress","load","error"),a={},l=new XMLHttpRequest,c=null;return!this.XDomainRequest||"withCredentials"in l||!/^(http(s)?:)?\/\//.test(n)||(l=new XDomainRequest),"onload"in l?l.onload=l.onerror=i:l.onreadystatechange=function(){l.readyState>3&&i()},l.onprogress=function(n){var t=ao.event;ao.event=n;try{o.progress.call(u,l)}finally{ao.event=t}},u.header=function(n,t){return n=(n+"").toLowerCase(),arguments.length<2?a[n]:(null==t?delete a[n]:a[n]=t+"",u)},u.mimeType=function(n){return arguments.length?(t=null==n?null:n+"",u):t},u.responseType=function(n){return arguments.length?(c=n,u):c},u.response=function(n){return e=n,u},["get","post"].forEach(function(n){u[n]=function(){return u.send.apply(u,[n].concat(co(arguments)))}}),u.send=function(e,r,i){if(
2===arguments.length&&"function"==typeof r&&(i=r,r=null),l.open(e,n,!0),null==t||"accept"in a||(a.accept=t+",*/*"),l.setRequestHeader)for(var f in a)l.setRequestHeader(f,a[f]);return null!=t&&l.overrideMimeType&&l.overrideMimeType(t),null!=c&&(l.responseType=c),null!=i&&u.on("error",i).on("load",function(n){i(null,n)}),o.beforesend.call(u,l),l.send(null==r?null:r),u},u.abort=function(){return l.abort(),u},ao.rebind(u,o,"on"),null==r?u:u.get(zn(r))}function zn(n){return 1===n.length?function(t,e){n(null==t?e:null)}:n}function Ln(n){var t=n.responseType;return t&&"text"!==t?n.response:n.responseText}function qn(n,t,e){var r=arguments.length;2>r&&(t=0),3>r&&(e=Date.now());var i=e+t,u={c:n,t:i,n:null};return aa?aa.n=u:oa=u,aa=u,la||(ca=clearTimeout(ca),la=1,fa(Tn)),u}function Tn(){var n=Rn(),t=Dn()-n;t>24?(isFinite(t)&&(clearTimeout(ca),ca=setTimeout(Tn,t)),la=0):(la=1,fa(Tn))}function Rn(){for(var n=Date.now(),t=oa;t;)n>=t.t&&t.c(n-t.t)&&(t.c=null),t=t.n;return n}function Dn(){for(var
n,t=oa,e=1/0;t;)t.c?(t.t<e&&(e=t.t),t=(n=t).n):t=n?n.n=t.n:oa=t.n;return aa=n,e}function Pn(n,t){return t-(n?Math.ceil(Math.log(n)/Math.LN10):1)}function Un(n,t){var e=Math.pow(10,3*xo(8-t));return{scale:t>8?function(n){return n/e}:function(n){return n*e},symbol:n}}function jn(n){var t=n.decimal,e=n.thousands,r=n.grouping,i=n.currency,u=r&&e?function(n,t){for(var i=n.length,u=[],o=0,a=r[0],l=0;i>0&&a>0&&(l+a+1>t&&(a=Math.max(1,t-l)),u.push(n.substring(i-=a,i+a)),!((l+=a+1)>t));)a=r[o=(o+1)%r.length];return u.reverse().join(e)}:m;return function(n){var e=ha.exec(n),r=e[1]||" ",o=e[2]||">",a=e[3]||"-",l=e[4]||"",c=e[5],f=+e[6],s=e[7],h=e[8],p=e[9],g=1,v="",d="",y=!1,m=!0;switch(h&&(h=+h.substring(1)),(c||"0"===r&&"="===o)&&(c=r="0",o="="),p){case"n":s=!0,p="g";break;case"%":g=100,d="%",p="f";break;case"p":g=100,d="%",p="r";break;case"b":case"o":case"x":case"X":"#"===l&&(v="0"+p.toLowerCase());case"c":m=!1;case"d":y=!0,h=0;break;case"s":g=-1,p="r"}"$"===l&&(v=i[0],d=i[1]),"r"!=p||h||(p
="g"),null!=h&&("g"==p?h=Math.max(1,Math.min(21,h)):"e"!=p&&"f"!=p||(h=Math.max(0,Math.min(20,h)))),p=pa.get(p)||Fn;var M=c&&s;return function(n){var e=d;if(y&&n%1)return"";var i=0>n||0===n&&0>1/n?(n=-n,"-"):"-"===a?"":a;if(0>g){var l=ao.formatPrefix(n,h);n=l.scale(n),e=l.symbol+d}else n*=g;n=p(n,h);var x,b,_=n.lastIndexOf(".");if(0>_){var w=m?n.lastIndexOf("e"):-1;0>w?(x=n,b=""):(x=n.substring(0,w),b=n.substring(w))}else x=n.substring(0,_),b=t+n.substring(_+1);!c&&s&&(x=u(x,1/0));var S=v.length+x.length+b.length+(M?0:i.length),k=f>S?new Array(S=f-S+1).join(r):"";return M&&(x=u(k+x,k.length?f-b.length:1/0)),i+=v,n=x+b,("<"===o?i+n+k:">"===o?k+i+n:"^"===o?k.substring(0,S>>=1)+i+n+k.substring(S):i+(M?n:k+n))+e}}}function Fn(n){return n+""}function Hn(){this._=new Date(arguments.length>1?Date.UTC.apply(this,arguments):arguments[0])}function On(n,t,e){function r(t){var e=n(t),r=u(e,1);return r-t>t-e?e:r}function i(e){return t(e=n(new va(e-1)),1),e}function u(n,e){return t(n=new va(+n),e
),n}function o(n,r,u){var o=i(n),a=[];if(u>1)for(;r>o;)e(o)%u||a.push(new Date(+o)),t(o,1);else for(;r>o;)a.push(new Date(+o)),t(o,1);return a}function a(n,t,e){try{va=Hn;var r=new Hn;return r._=n,o(r,t,e)}finally{va=Date}}n.floor=n,n.round=r,n.ceil=i,n.offset=u,n.range=o;var l=n.utc=In(n);return l.floor=l,l.round=In(r),l.ceil=In(i),l.offset=In(u),l.range=a,n}function In(n){return function(t,e){try{va=Hn;var r=new Hn;return r._=t,n(r,e)._}finally{va=Date}}}function Yn(n){function t(n){function t(t){for(var e,i,u,o=[],a=-1,l=0;++a<r;)37===n.charCodeAt(a)&&(o.push(n.slice(l,a)),null!=(i=ya[e=n.charAt(++a)])&&(e=n.charAt(++a)),(u=A[e])&&(e=u(t,null==i?"e"===e?" ":"0":i)),o.push(e),l=a+1);return o.push(n.slice(l,a)),o.join("")}var r=n.length;return t.parse=function(t){var r={y:1900,m:0,d:1,H:0,M:0,S:0,L:0,Z:null},i=e(r,n,t,0);if(i!=t.length)return null;"p"in r&&(r.H=r.H%12+12*r.p);var u=null!=r.Z&&va!==Hn,o=new(u?Hn:va);return"j"in r?o.setFullYear(r.y,0,r.j):"W"in r||"U"in r?("w"in r||(
r.w="W"in r?1:0),o.setFullYear(r.y,0,1),o.setFullYear(r.y,0,"W"in r?(r.w+6)%7+7*r.W-(o.getDay()+5)%7:r.w+7*r.U-(o.getDay()+6)%7)):o.setFullYear(r.y,r.m,r.d),o.setHours(r.H+(r.Z/100|0),r.M+r.Z%100,r.S,r.L),u?o._:o},t.toString=function(){return n},t}function e(n,t,e,r){for(var i,u,o,a=0,l=t.length,c=e.length;l>a;){if(r>=c)return-1;if(i=t.charCodeAt(a++),37===i){if(o=t.charAt(a++),u=C[o in ya?t.charAt(a++):o],!u||(r=u(n,e,r))<0)return-1}else if(i!=e.charCodeAt(r++))return-1}return r}function r(n,t,e){_.lastIndex=0;var r=_.exec(t.slice(e));return r?(n.w=w.get(r[0].toLowerCase()),e+r[0].length):-1}function i(n,t,e){x.lastIndex=0;var r=x.exec(t.slice(e));return r?(n.w=b.get(r[0].toLowerCase()),e+r[0].length):-1}function u(n,t,e){N.lastIndex=0;var r=N.exec(t.slice(e));return r?(n.m=E.get(r[0].toLowerCase()),e+r[0].length):-1}function o(n,t,e){S.lastIndex=0;var r=S.exec(t.slice(e));return r?(n.m=k.get(r[0].toLowerCase()),e+r[0].length):-1}function a(n,t,r){return e(n,A.c.toString(),t,r)}fun
ction l(n,t,r){return e(n,A.x.toString(),t,r)}function c(n,t,r){return e(n,A.X.toString(),t,r)}function f(n,t,e){var r=M.get(t.slice(e,e+=2).toLowerCase());return null==r?-1:(n.p=r,e)}var s=n.dateTime,h=n.date,p=n.time,g=n.periods,v=n.days,d=n.shortDays,y=n.months,m=n.shortMonths;t.utc=function(n){function e(n){try{va=Hn;var t=new va;return t._=n,r(t)}finally{va=Date}}var r=t(n);return e.parse=function(n){try{va=Hn;var t=r.parse(n);return t&&t._}finally{va=Date}},e.toString=r.toString,e},t.multi=t.utc.multi=ct;var M=ao.map(),x=Vn(v),b=Xn(v),_=Vn(d),w=Xn(d),S=Vn(y),k=Xn(y),N=Vn(m),E=Xn(m);g.forEach(function(n,t){M.set(n.toLowerCase(),t)});var A={a:function(n){return d[n.getDay()]},A:function(n){return v[n.getDay()]},b:function(n){return m[n.getMonth()]},B:function(n){return y[n.getMonth()]},c:t(s),d:function(n,t){return Zn(n.getDate(),t,2)},e:function(n,t){return Zn(n.getDate(),t,2)},H:function(n,t){return Zn(n.getHours(),t,2)},I:function(n,t){return Zn(n.getHours()%12||12,t,2)},j:fu
nction(n,t){return Zn(1+ga.dayOfYear(n),t,3)},L:function(n,t){return Zn(n.getMilliseconds(),t,3)},m:function(n,t){return Zn(n.getMonth()+1,t,2)},M:function(n,t){return Zn(n.getMinutes(),t,2)},p:function(n){return g[+(n.getHours()>=12)]},S:function(n,t){return Zn(n.getSeconds(),t,2)},U:function(n,t){return Zn(ga.sundayOfYear(n),t,2)},w:function(n){return n.getDay()},W:function(n,t){return Zn(ga.mondayOfYear(n),t,2)},x:t(h),X:t(p),y:function(n,t){return Zn(n.getFullYear()%100,t,2)},Y:function(n,t){return Zn(n.getFullYear()%1e4,t,4)},Z:at,"%":function(){return"%"}},C={a:r,A:i,b:u,B:o,c:a,d:tt,e:tt,H:rt,I:rt,j:et,L:ot,m:nt,M:it,p:f,S:ut,U:Bn,w:$n,W:Wn,x:l,X:c,y:Gn,Y:Jn,Z:Kn,"%":lt};return t}function Zn(n,t,e){var r=0>n?"-":"",i=(r?-n:n)+"",u=i.length;return r+(e>u?new Array(e-u+1).join(t)+i:i)}function Vn(n){return new RegExp("^(?:"+n.map(ao.requote).join("|")+")","i")}function Xn(n){for(var t=new c,e=-1,r=n.length;++e<r;)t.set(n[e].toLowerCase(),e);return t}function $n(n,t,e){ma.lastIn
dex=0;var r=ma.exec(t.slice(e,e+1));return r?(n.w=+r[0],e+r[0].length):-1}function Bn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.U=+r[0],e+r[0].length):-1}function Wn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e));return r?(n.W=+r[0],e+r[0].length):-1}function Jn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+4));return r?(n.y=+r[0],e+r[0].length):-1}function Gn(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.y=Qn(+r[0]),e+r[0].length):-1}function Kn(n,t,e){return/^[+-]\d{4}$/.test(t=t.slice(e,e+5))?(n.Z=-t,e+5):-1}function Qn(n){return n+(n>68?1900:2e3)}function nt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.m=r[0]-1,e+r[0].length):-1}function tt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.d=+r[0],e+r[0].length):-1}function et(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.j=+r[0],e+r[0].length):-1}function rt(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.H=+r[0],e+r[0].len
gth):-1}function it(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.M=+r[0],e+r[0].length):-1}function ut(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+2));return r?(n.S=+r[0],e+r[0].length):-1}function ot(n,t,e){ma.lastIndex=0;var r=ma.exec(t.slice(e,e+3));return r?(n.L=+r[0],e+r[0].length):-1}function at(n){var t=n.getTimezoneOffset(),e=t>0?"-":"+",r=xo(t)/60|0,i=xo(t)%60;return e+Zn(r,"0",2)+Zn(i,"0",2)}function lt(n,t,e){Ma.lastIndex=0;var r=Ma.exec(t.slice(e,e+1));return r?e+r[0].length:-1}function ct(n){for(var t=n.length,e=-1;++e<t;)n[e][0]=this(n[e][0]);return function(t){for(var e=0,r=n[e];!r[1](t);)r=n[++e];return r[0](t)}}function ft(){}function st(n,t,e){var r=e.s=n+t,i=r-n,u=r-i;e.t=n-u+(t-i)}function ht(n,t){n&&wa.hasOwnProperty(n.type)&&wa[n.type](n,t)}function pt(n,t,e){var r,i=-1,u=n.length-e;for(t.lineStart();++i<u;)r=n[i],t.point(r[0],r[1],r[2]);t.lineEnd()}function gt(n,t){var e=-1,r=n.length;for(t.polygonStart();++e<r;)pt(n[e],t,1);t.polygonEnd
()}function vt(){function n(n,t){n*=Yo,t=t*Yo/2+Fo/4;var e=n-r,o=e>=0?1:-1,a=o*e,l=Math.cos(t),c=Math.sin(t),f=u*c,s=i*l+f*Math.cos(a),h=f*o*Math.sin(a);ka.add(Math.atan2(h,s)),r=n,i=l,u=c}var t,e,r,i,u;Na.point=function(o,a){Na.point=n,r=(t=o)*Yo,i=Math.cos(a=(e=a)*Yo/2+Fo/4),u=Math.sin(a)},Na.lineEnd=function(){n(t,e)}}function dt(n){var t=n[0],e=n[1],r=Math.cos(e);return[r*Math.cos(t),r*Math.sin(t),Math.sin(e)]}function yt(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]}function mt(n,t){return[n[1]*t[2]-n[2]*t[1],n[2]*t[0]-n[0]*t[2],n[0]*t[1]-n[1]*t[0]]}function Mt(n,t){n[0]+=t[0],n[1]+=t[1],n[2]+=t[2]}function xt(n,t){return[n[0]*t,n[1]*t,n[2]*t]}function bt(n){var t=Math.sqrt(n[0]*n[0]+n[1]*n[1]+n[2]*n[2]);n[0]/=t,n[1]/=t,n[2]/=t}function _t(n){return[Math.atan2(n[1],n[0]),tn(n[2])]}function wt(n,t){return xo(n[0]-t[0])<Uo&&xo(n[1]-t[1])<Uo}function St(n,t){n*=Yo;var e=Math.cos(t*=Yo);kt(e*Math.cos(n),e*Math.sin(n),Math.sin(t))}function kt(n,t,e){++Ea,Ca+=(n-Ca)/Ea,za+=(t-za)/Ea,La+=
(e-La)/Ea}function Nt(){function n(n,i){n*=Yo;var u=Math.cos(i*=Yo),o=u*Math.cos(n),a=u*Math.sin(n),l=Math.sin(i),c=Math.atan2(Math.sqrt((c=e*l-r*a)*c+(c=r*o-t*l)*c+(c=t*a-e*o)*c),t*o+e*a+r*l);Aa+=c,qa+=c*(t+(t=o)),Ta+=c*(e+(e=a)),Ra+=c*(r+(r=l)),kt(t,e,r)}var t,e,r;ja.point=function(i,u){i*=Yo;var o=Math.cos(u*=Yo);t=o*Math.cos(i),e=o*Math.sin(i),r=Math.sin(u),ja.point=n,kt(t,e,r)}}function Et(){ja.point=St}function At(){function n(n,t){n*=Yo;var e=Math.cos(t*=Yo),o=e*Math.cos(n),a=e*Math.sin(n),l=Math.sin(t),c=i*l-u*a,f=u*o-r*l,s=r*a-i*o,h=Math.sqrt(c*c+f*f+s*s),p=r*o+i*a+u*l,g=h&&-nn(p)/h,v=Math.atan2(h,p);Da+=g*c,Pa+=g*f,Ua+=g*s,Aa+=v,qa+=v*(r+(r=o)),Ta+=v*(i+(i=a)),Ra+=v*(u+(u=l)),kt(r,i,u)}var t,e,r,i,u;ja.point=function(o,a){t=o,e=a,ja.point=n,o*=Yo;var l=Math.cos(a*=Yo);r=l*Math.cos(o),i=l*Math.sin(o),u=Math.sin(a),kt(r,i,u)},ja.lineEnd=function(){n(t,e),ja.lineEnd=Et,ja.point=St}}function Ct(n,t){function e(e,r){return e=n(e,r),t(e[0],e[1])}return n.invert&&t.invert&&(e.inv
ert=function(e,r){return e=t.invert(e,r),e&&n.invert(e[0],e[1])}),e}function zt(){return!0}function Lt(n,t,e,r,i){var u=[],o=[];if(n.forEach(function(n){if(!((t=n.length-1)<=0)){var t,e=n[0],r=n[t];if(wt(e,r)){i.lineStart();for(var a=0;t>a;++a)i.point((e=n[a])[0],e[1]);return void i.lineEnd()}var l=new Tt(e,n,null,!0),c=new Tt(e,null,l,!1);l.o=c,u.push(l),o.push(c),l=new Tt(r,n,null,!1),c=new Tt(r,null,l,!0),l.o=c,u.push(l),o.push(c)}}),o.sort(t),qt(u),qt(o),u.length){for(var a=0,l=e,c=o.length;c>a;++a)o[a].e=l=!l;for(var f,s,h=u[0];;){for(var p=h,g=!0;p.v;)if((p=p.n)===h)return;f=p.z,i.lineStart();do{if(p.v=p.o.v=!0,p.e){if(g)for(var a=0,c=f.length;c>a;++a)i.point((s=f[a])[0],s[1]);else r(p.x,p.n.x,1,i);p=p.n}else{if(g){f=p.p.z;for(var a=f.length-1;a>=0;--a)i.point((s=f[a])[0],s[1])}else r(p.x,p.p.x,-1,i);p=p.p}p=p.o,f=p.z,g=!g}while(!p.v);i.lineEnd()}}}function qt(n){if(t=n.length){for(var t,e,r=0,i=n[0];++r<t;)i.n=e=n[r],e.p=i,i=e;i.n=e=n[0],e.p=i}}function Tt(n,t,e,r){this.x=n,t
his.z=t,this.o=e,this.e=r,this.v=!1,this.n=this.p=null}function Rt(n,t,e,r){return function(i,u){function o(t,e){var r=i(t,e);n(t=r[0],e=r[1])&&u.point(t,e)}function a(n,t){var e=i(n,t);d.point(e[0],e[1])}function l(){m.point=a,d.lineStart()}function c(){m.point=o,d.lineEnd()}function f(n,t){v.push([n,t]);var e=i(n,t);x.point(e[0],e[1])}function s(){x.lineStart(),v=[]}function h(){f(v[0][0],v[0][1]),x.lineEnd();var n,t=x.clean(),e=M.buffer(),r=e.length;if(v.pop(),g.push(v),v=null,r)if(1&t){n=e[0];var i,r=n.length-1,o=-1;if(r>0){for(b||(u.polygonStart(),b=!0),u.lineStart();++o<r;)u.point((i=n[o])[0],i[1]);u.lineEnd()}}else r>1&&2&t&&e.push(e.pop().concat(e.shift())),p.push(e.filter(Dt))}var p,g,v,d=t(u),y=i.invert(r[0],r[1]),m={point:o,lineStart:l,lineEnd:c,polygonStart:function(){m.point=f,m.lineStart=s,m.lineEnd=h,p=[],g=[]},polygonEnd:function(){m.point=o,m.lineStart=l,m.lineEnd=c,p=ao.merge(p);var n=Ot(y,g);p.length?(b||(u.polygonStart(),b=!0),Lt(p,Ut,n,e,u)):n&&(b||(u.polygonSta
rt(),b=!0),u.lineStart(),e(null,null,1,u),u.lineEnd()),b&&(u.polygonEnd(),b=!1),p=g=null},sphere:function(){u.polygonStart(),u.lineStart(),e(null,null,1,u),u.lineEnd(),u.polygonEnd()}},M=Pt(),x=t(M),b=!1;return m}}function Dt(n){return n.length>1}function Pt(){var n,t=[];return{lineStart:function(){t.push(n=[])},point:function(t,e){n.push([t,e])},lineEnd:b,buffer:function(){var e=t;return t=[],n=null,e},rejoin:function(){t.length>1&&t.push(t.pop().concat(t.shift()))}}}function Ut(n,t){return((n=n.x)[0]<0?n[1]-Io-Uo:Io-n[1])-((t=t.x)[0]<0?t[1]-Io-Uo:Io-t[1])}function jt(n){var t,e=NaN,r=NaN,i=NaN;return{lineStart:function(){n.lineStart(),t=1},point:function(u,o){var a=u>0?Fo:-Fo,l=xo(u-e);xo(l-Fo)<Uo?(n.point(e,r=(r+o)/2>0?Io:-Io),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),n.point(u,r),t=0):i!==a&&l>=Fo&&(xo(e-i)<Uo&&(e-=i*Uo),xo(u-a)<Uo&&(u-=a*Uo),r=Ft(e,r,u,o),n.point(i,r),n.lineEnd(),n.lineStart(),n.point(a,r),t=0),n.point(e=u,r=o),i=a},lineEnd:function(){n.lineEnd(),e=r=
NaN},clean:function(){return 2-t}}}function Ft(n,t,e,r){var i,u,o=Math.sin(n-e);return xo(o)>Uo?Math.atan((Math.sin(t)*(u=Math.cos(r))*Math.sin(e)-Math.sin(r)*(i=Math.cos(t))*Math.sin(n))/(i*u*o)):(t+r)/2}function Ht(n,t,e,r){var i;if(null==n)i=e*Io,r.point(-Fo,i),r.point(0,i),r.point(Fo,i),r.point(Fo,0),r.point(Fo,-i),r.point(0,-i),r.point(-Fo,-i),r.point(-Fo,0),r.point(-Fo,i);else if(xo(n[0]-t[0])>Uo){var u=n[0]<t[0]?Fo:-Fo;i=e*u/2,r.point(-u,i),r.point(0,i),r.point(u,i)}else r.point(t[0],t[1])}function Ot(n,t){var e=n[0],r=n[1],i=[Math.sin(e),-Math.cos(e),0],u=0,o=0;ka.reset();for(var a=0,l=t.length;l>a;++a){var c=t[a],f=c.length;if(f)for(var s=c[0],h=s[0],p=s[1]/2+Fo/4,g=Math.sin(p),v=Math.cos(p),d=1;;){d===f&&(d=0),n=c[d];var y=n[0],m=n[1]/2+Fo/4,M=Math.sin(m),x=Math.cos(m),b=y-h,_=b>=0?1:-1,w=_*b,S=w>Fo,k=g*M;if(ka.add(Math.atan2(k*_*Math.sin(w),v*x+k*Math.cos(w))),u+=S?b+_*Ho:b,S^h>=e^y>=e){var N=mt(dt(s),dt(n));bt(N);var E=mt(i,N);bt(E);var A=(S^b>=0?-1:1)*tn(E[2]);(r>A||r==
=A&&(N[0]||N[1]))&&(o+=S^b>=0?1:-1)}if(!d++)break;h=y,g=M,v=x,s=n}}return(-Uo>u||Uo>u&&-Uo>ka)^1&o}function It(n){function t(n,t){return Math.cos(n)*Math.cos(t)>u}function e(n){var e,u,l,c,f;return{lineStart:function(){c=l=!1,f=1},point:function(s,h){var p,g=[s,h],v=t(s,h),d=o?v?0:i(s,h):v?i(s+(0>s?Fo:-Fo),h):0;if(!e&&(c=l=v)&&n.lineStart(),v!==l&&(p=r(e,g),(wt(e,p)||wt(g,p))&&(g[0]+=Uo,g[1]+=Uo,v=t(g[0],g[1]))),v!==l)f=0,v?(n.lineStart(),p=r(g,e),n.point(p[0],p[1])):(p=r(e,g),n.point(p[0],p[1]),n.lineEnd()),e=p;else if(a&&e&&o^v){var y;d&u||!(y=r(g,e,!0))||(f=0,o?(n.lineStart(),n.point(y[0][0],y[0][1]),n.point(y[1][0],y[1][1]),n.lineEnd()):(n.point(y[1][0],y[1][1]),n.lineEnd(),n.lineStart(),n.point(y[0][0],y[0][1])))}!v||e&&wt(e,g)||n.point(g[0],g[1]),e=g,l=v,u=d},lineEnd:function(){l&&n.lineEnd(),e=null},clean:function(){return f|(c&&l)<<1}}}function r(n,t,e){var r=dt(n),i=dt(t),o=[1,0,0],a=mt(r,i),l=yt(a,a),c=a[0],f=l-c*c;if(!f)return!e&&n;var s=u*l/f,h=-u*c/f,p=mt(o,a),g=xt(o,s)
,v=xt(a,h);Mt(g,v);var d=p,y=yt(g,d),m=yt(d,d),M=y*y-m*(yt(g,g)-1);if(!(0>M)){var x=Math.sqrt(M),b=xt(d,(-y-x)/m);if(Mt(b,g),b=_t(b),!e)return b;var _,w=n[0],S=t[0],k=n[1],N=t[1];w>S&&(_=w,w=S,S=_);var E=S-w,A=xo(E-Fo)<Uo,C=A||Uo>E;if(!A&&k>N&&(_=k,k=N,N=_),C?A?k+N>0^b[1]<(xo(b[0]-w)<Uo?k:N):k<=b[1]&&b[1]<=N:E>Fo^(w<=b[0]&&b[0]<=S)){var z=xt(d,(-y+x)/m);return Mt(z,g),[b,_t(z)]}}}function i(t,e){var r=o?n:Fo-n,i=0;return-r>t?i|=1:t>r&&(i|=2),-r>e?i|=4:e>r&&(i|=8),i}var u=Math.cos(n),o=u>0,a=xo(u)>Uo,l=ve(n,6*Yo);return Rt(t,e,l,o?[0,-n]:[-Fo,n-Fo])}function Yt(n,t,e,r){return function(i){var u,o=i.a,a=i.b,l=o.x,c=o.y,f=a.x,s=a.y,h=0,p=1,g=f-l,v=s-c;if(u=n-l,g||!(u>0)){if(u/=g,0>g){if(h>u)return;p>u&&(p=u)}else if(g>0){if(u>p)return;u>h&&(h=u)}if(u=e-l,g||!(0>u)){if(u/=g,0>g){if(u>p)return;u>h&&(h=u)}else if(g>0){if(h>u)return;p>u&&(p=u)}if(u=t-c,v||!(u>0)){if(u/=v,0>v){if(h>u)return;p>u&&(p=u)}else if(v>0){if(u>p)return;u>h&&(h=u)}if(u=r-c,v||!(0>u)){if(u/=v,0>v){if(u>p)return;u>h&&
(h=u)}else if(v>0){if(h>u)return;p>u&&(p=u)}return h>0&&(i.a={x:l+h*g,y:c+h*v}),1>p&&(i.b={x:l+p*g,y:c+p*v}),i}}}}}}function Zt(n,t,e,r){function i(r,i){return xo(r[0]-n)<Uo?i>0?0:3:xo(r[0]-e)<Uo?i>0?2:1:xo(r[1]-t)<Uo?i>0?1:0:i>0?3:2}function u(n,t){return o(n.x,t.x)}function o(n,t){var e=i(n,1),r=i(t,1);return e!==r?e-r:0===e?t[1]-n[1]:1===e?n[0]-t[0]:2===e?n[1]-t[1]:t[0]-n[0]}return function(a){function l(n){for(var t=0,e=d.length,r=n[1],i=0;e>i;++i)for(var u,o=1,a=d[i],l=a.length,c=a[0];l>o;++o)u=a[o],c[1]<=r?u[1]>r&&Q(c,u,n)>0&&++t:u[1]<=r&&Q(c,u,n)<0&&--t,c=u;return 0!==t}function c(u,a,l,c){var f=0,s=0;if(null==u||(f=i(u,l))!==(s=i(a,l))||o(u,a)<0^l>0){do c.point(0===f||3===f?n:e,f>1?r:t);while((f=(f+l+4)%4)!==s)}else c.point(a[0],a[1])}function f(i,u){return i>=n&&e>=i&&u>=t&&r>=u}function s(n,t){f(n,t)&&a.point(n,t)}function h(){C.point=g,d&&d.push(y=[]),S=!0,w=!1,b=_=NaN}function p(){v&&(g(m,M),x&&w&&E.rejoin(),v.push(E.buffer())),C.point=s,w&&a.lineEnd()}function g(n,t){n=
Math.max(-Ha,Math.min(Ha,n)),t=Math.max(-Ha,Math.min(Ha,t));var e=f(n,t);if(d&&y.push([n,t]),S)m=n,M=t,x=e,S=!1,e&&(a.lineStart(),a.point(n,t));else if(e&&w)a.point(n,t);else{var r={a:{x:b,y:_},b:{x:n,y:t}};A(r)?(w||(a.lineStart(),a.point(r.a.x,r.a.y)),a.point(r.b.x,r.b.y),e||a.lineEnd(),k=!1):e&&(a.lineStart(),a.point(n,t),k=!1)}b=n,_=t,w=e}var v,d,y,m,M,x,b,_,w,S,k,N=a,E=Pt(),A=Yt(n,t,e,r),C={point:s,lineStart:h,lineEnd:p,polygonStart:function(){a=E,v=[],d=[],k=!0},polygonEnd:function(){a=N,v=ao.merge(v);var t=l([n,r]),e=k&&t,i=v.length;(e||i)&&(a.polygonStart(),e&&(a.lineStart(),c(null,null,1,a),a.lineEnd()),i&&Lt(v,u,t,c,a),a.polygonEnd()),v=d=y=null}};return C}}function Vt(n){var t=0,e=Fo/3,r=ae(n),i=r(t,e);return i.parallels=function(n){return arguments.length?r(t=n[0]*Fo/180,e=n[1]*Fo/180):[t/Fo*180,e/Fo*180]},i}function Xt(n,t){function e(n,t){var e=Math.sqrt(u-2*i*Math.sin(t))/i;return[e*Math.sin(n*=i),o-e*Math.cos(n)]}var r=Math.sin(n),i=(r+Math.sin(t))/2,u=1+r*(2*i-r),o=M
ath.sqrt(u)/i;return e.invert=function(n,t){var e=o-t;return[Math.atan2(n,e)/i,tn((u-(n*n+e*e)*i*i)/(2*i))]},e}function $t(){function n(n,t){Ia+=i*n-r*t,r=n,i=t}var t,e,r,i;$a.point=function(u,o){$a.point=n,t=r=u,e=i=o},$a.lineEnd=function(){n(t,e)}}function Bt(n,t){Ya>n&&(Ya=n),n>Va&&(Va=n),Za>t&&(Za=t),t>Xa&&(Xa=t)}function Wt(){function n(n,t){o.push("M",n,",",t,u)}function t(n,t){o.push("M",n,",",t),a.point=e}function e(n,t){o.push("L",n,",",t)}function r(){a.point=n}function i(){o.push("Z")}var u=Jt(4.5),o=[],a={point:n,lineStart:function(){a.point=t},lineEnd:r,polygonStart:function(){a.lineEnd=i},polygonEnd:function(){a.lineEnd=r,a.point=n},pointRadius:function(n){return u=Jt(n),a},result:function(){if(o.length){var n=o.join("");return o=[],n}}};return a}function Jt(n){return"m0,"+n+"a"+n+","+n+" 0 1,1 0,"+-2*n+"a"+n+","+n+" 0 1,1 0,"+2*n+"z"}function Gt(n,t){Ca+=n,za+=t,++La}function Kt(){function n(n,r){var i=n-t,u=r-e,o=Math.sqrt(i*i+u*u);qa+=o*(t+n)/2,Ta+=o*(e+r)/2,Ra+=o,G
t(t=n,e=r)}var t,e;Wa.point=function(r,i){Wa.point=n,Gt(t=r,e=i)}}function Qt(){Wa.point=Gt}function ne(){function n(n,t){var e=n-r,u=t-i,o=Math.sqrt(e*e+u*u);qa+=o*(r+n)/2,Ta+=o*(i+t)/2,Ra+=o,o=i*n-r*t,Da+=o*(r+n),Pa+=o*(i+t),Ua+=3*o,Gt(r=n,i=t)}var t,e,r,i;Wa.point=function(u,o){Wa.point=n,Gt(t=r=u,e=i=o)},Wa.lineEnd=function(){n(t,e)}}function te(n){function t(t,e){n.moveTo(t+o,e),n.arc(t,e,o,0,Ho)}function e(t,e){n.moveTo(t,e),a.point=r}function r(t,e){n.lineTo(t,e)}function i(){a.point=t}function u(){n.closePath()}var o=4.5,a={point:t,lineStart:function(){a.point=e},lineEnd:i,polygonStart:function(){a.lineEnd=u},polygonEnd:function(){a.lineEnd=i,a.point=t},pointRadius:function(n){return o=n,a},result:b};return a}function ee(n){function t(n){return(a?r:e)(n)}function e(t){return ue(t,function(e,r){e=n(e,r),t.point(e[0],e[1])})}function r(t){function e(e,r){e=n(e,r),t.point(e[0],e[1])}function r(){M=NaN,S.point=u,t.lineStart()}function u(e,r){var u=dt([e,r]),o=n(e,r);i(M,x,m,b,_,
w,M=o[0],x=o[1],m=e,b=u[0],_=u[1],w=u[2],a,t),t.point(M,x)}function o(){S.point=e,t.lineEnd()}function l(){
-r(),S.point=c,S.lineEnd=f}function c(n,t){u(s=n,h=t),p=M,g=x,v=b,d=_,y=w,S.point=u}function f(){i(M,x,m,b,_,w,p,g,s,v,d,y,a,t),S.lineEnd=o,o()}var s,h,p,g,v,d,y,m,M,x,b,_,w,S={point:e,lineStart:r,lineEnd:o,polygonStart:function(){t.polygonStart(),S.lineStart=l},polygonEnd:function(){t.polygonEnd(),S.lineStart=r}};return S}function i(t,e,r,a,l,c,f,s,h,p,g,v,d,y){var m=f-t,M=s-e,x=m*m+M*M;if(x>4*u&&d--){var b=a+p,_=l+g,w=c+v,S=Math.sqrt(b*b+_*_+w*w),k=Math.asin(w/=S),N=xo(xo(w)-1)<Uo||xo(r-h)<Uo?(r+h)/2:Math.atan2(_,b),E=n(N,k),A=E[0],C=E[1],z=A-t,L=C-e,q=M*z-m*L;(q*q/x>u||xo((m*z+M*L)/x-.5)>.3||o>a*p+l*g+c*v)&&(i(t,e,r,a,l,c,A,C,N,b/=S,_/=S,w,d,y),y.point(A,C),i(A,C,N,b,_,w,f,s,h,p,g,v,d,y))}}var u=.5,o=Math.cos(30*Yo),a=16;return t.precision=function(n){return arguments.length?(a=(u=n*n)>0&&16,t):Math.sqrt(u)},t}function re(n){var t=ee(function(t,e){return n([t*Zo,e*Zo])});return function(n){return le(t(n))}}function ie(n){this.stream=n}function ue(n,t){return{point:t,sphere:functio
n(){n.sphere()},lineStart:function(){n.lineStart()},lineEnd:function(){n.lineEnd()},polygonStart:function(){n.polygonStart()},polygonEnd:function(){n.polygonEnd()}}}function oe(n){return ae(function(){return n})()}function ae(n){function t(n){return n=a(n[0]*Yo,n[1]*Yo),[n[0]*h+l,c-n[1]*h]}function e(n){return n=a.invert((n[0]-l)/h,(c-n[1])/h),n&&[n[0]*Zo,n[1]*Zo]}function r(){a=Ct(o=se(y,M,x),u);var n=u(v,d);return l=p-n[0]*h,c=g+n[1]*h,i()}function i(){return f&&(f.valid=!1,f=null),t}var u,o,a,l,c,f,s=ee(function(n,t){return n=u(n,t),[n[0]*h+l,c-n[1]*h]}),h=150,p=480,g=250,v=0,d=0,y=0,M=0,x=0,b=Fa,_=m,w=null,S=null;return t.stream=function(n){return f&&(f.valid=!1),f=le(b(o,s(_(n)))),f.valid=!0,f},t.clipAngle=function(n){return arguments.length?(b=null==n?(w=n,Fa):It((w=+n)*Yo),i()):w},t.clipExtent=function(n){return arguments.length?(S=n,_=n?Zt(n[0][0],n[0][1],n[1][0],n[1][1]):m,i()):S},t.scale=function(n){return arguments.length?(h=+n,r()):h},t.translate=function(n){return argum
ents.length?(p=+n[0],g=+n[1],r()):[p,g]},t.center=function(n){return arguments.length?(v=n[0]%360*Yo,d=n[1]%360*Yo,r()):[v*Zo,d*Zo]},t.rotate=function(n){return arguments.length?(y=n[0]%360*Yo,M=n[1]%360*Yo,x=n.length>2?n[2]%360*Yo:0,r()):[y*Zo,M*Zo,x*Zo]},ao.rebind(t,s,"precision"),function(){return u=n.apply(this,arguments),t.invert=u.invert&&e,r()}}function le(n){return ue(n,function(t,e){n.point(t*Yo,e*Yo)})}function ce(n,t){return[n,t]}function fe(n,t){return[n>Fo?n-Ho:-Fo>n?n+Ho:n,t]}function se(n,t,e){return n?t||e?Ct(pe(n),ge(t,e)):pe(n):t||e?ge(t,e):fe}function he(n){return function(t,e){return t+=n,[t>Fo?t-Ho:-Fo>t?t+Ho:t,e]}}function pe(n){var t=he(n);return t.invert=he(-n),t}function ge(n,t){function e(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math.sin(t),f=c*r+a*i;return[Math.atan2(l*u-f*o,a*r-c*i),tn(f*u+l*o)]}var r=Math.cos(n),i=Math.sin(n),u=Math.cos(t),o=Math.sin(t);return e.invert=function(n,t){var e=Math.cos(t),a=Math.cos(n)*e,l=Math.sin(n)*e,c=Math
.sin(t),f=c*u-l*o;return[Math.atan2(l*u+c*o,a*r+f*i),tn(f*r-a*i)]},e}function ve(n,t){var e=Math.cos(n),r=Math.sin(n);return function(i,u,o,a){var l=o*t;null!=i?(i=de(e,i),u=de(e,u),(o>0?u>i:i>u)&&(i+=o*Ho)):(i=n+o*Ho,u=n-.5*l);for(var c,f=i;o>0?f>u:u>f;f-=l)a.point((c=_t([e,-r*Math.cos(f),-r*Math.sin(f)]))[0],c[1])}}function de(n,t){var e=dt(t);e[0]-=n,bt(e);var r=nn(-e[1]);return((-e[2]<0?-r:r)+2*Math.PI-Uo)%(2*Math.PI)}function ye(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[n,t]})}}function me(n,t,e){var r=ao.range(n,t-Uo,e).concat(t);return function(n){return r.map(function(t){return[t,n]})}}function Me(n){return n.source}function xe(n){return n.target}function be(n,t,e,r){var i=Math.cos(t),u=Math.sin(t),o=Math.cos(r),a=Math.sin(r),l=i*Math.cos(n),c=i*Math.sin(n),f=o*Math.cos(e),s=o*Math.sin(e),h=2*Math.asin(Math.sqrt(on(r-t)+i*o*on(e-n))),p=1/Math.sin(h),g=h?function(n){var t=Math.sin(n*=h)*p,e=Math.sin(h-n)*p,r=e*l+t*f,i=e*c+t*s
,o=e*u+t*a;return[Math.atan2(i,r)*Zo,Math.atan2(o,Math.sqrt(r*r+i*i))*Zo]}:function(){return[n*Zo,t*Zo]};return g.distance=h,g}function _e(){function n(n,i){var u=Math.sin(i*=Yo),o=Math.cos(i),a=xo((n*=Yo)-t),l=Math.cos(a);Ja+=Math.atan2(Math.sqrt((a=o*Math.sin(a))*a+(a=r*u-e*o*l)*a),e*u+r*o*l),t=n,e=u,r=o}var t,e,r;Ga.point=function(i,u){t=i*Yo,e=Math.sin(u*=Yo),r=Math.cos(u),Ga.point=n},Ga.lineEnd=function(){Ga.point=Ga.lineEnd=b}}function we(n,t){function e(t,e){var r=Math.cos(t),i=Math.cos(e),u=n(r*i);return[u*i*Math.sin(t),u*Math.sin(e)]}return e.invert=function(n,e){var r=Math.sqrt(n*n+e*e),i=t(r),u=Math.sin(i),o=Math.cos(i);return[Math.atan2(n*u,r*o),Math.asin(r&&e*u/r)]},e}function Se(n,t){function e(n,t){o>0?-Io+Uo>t&&(t=-Io+Uo):t>Io-Uo&&(t=Io-Uo);var e=o/Math.pow(i(t),u);return[e*Math.sin(u*n),o-e*Math.cos(u*n)]}var r=Math.cos(n),i=function(n){return Math.tan(Fo/4+n/2)},u=n===t?Math.sin(n):Math.log(r/Math.cos(t))/Math.log(i(t)/i(n)),o=r*Math.pow(i(n),u)/u;return u?(e.inver
t=function(n,t){var e=o-t,r=K(u)*Math.sqrt(n*n+e*e);return[Math.atan2(n,e)/u,2*Math.atan(Math.pow(o/r,1/u))-Io]},e):Ne}function ke(n,t){function e(n,t){var e=u-t;return[e*Math.sin(i*n),u-e*Math.cos(i*n)]}var r=Math.cos(n),i=n===t?Math.sin(n):(r-Math.cos(t))/(t-n),u=r/i+n;return xo(i)<Uo?ce:(e.invert=function(n,t){var e=u-t;return[Math.atan2(n,e)/i,u-K(i)*Math.sqrt(n*n+e*e)]},e)}function Ne(n,t){return[n,Math.log(Math.tan(Fo/4+t/2))]}function Ee(n){var t,e=oe(n),r=e.scale,i=e.translate,u=e.clipExtent;return e.scale=function(){var n=r.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.translate=function(){var n=i.apply(e,arguments);return n===e?t?e.clipExtent(null):e:n},e.clipExtent=function(n){var o=u.apply(e,arguments);if(o===e){if(t=null==n){var a=Fo*r(),l=i();u([[l[0]-a,l[1]-a],[l[0]+a,l[1]+a]])}}else t&&(o=null);return o},e.clipExtent(null)}function Ae(n,t){return[Math.log(Math.tan(Fo/4+t/2)),-n]}function Ce(n){return n[0]}function ze(n){return n[1]}function Le(n){for(va
r t=n.length,e=[0,1],r=2,i=2;t>i;i++){for(;r>1&&Q(n[e[r-2]],n[e[r-1]],n[i])<=0;)--r;e[r++]=i}return e.slice(0,r)}function qe(n,t){return n[0]-t[0]||n[1]-t[1]}function Te(n,t,e){return(e[0]-t[0])*(n[1]-t[1])<(e[1]-t[1])*(n[0]-t[0])}function Re(n,t,e,r){var i=n[0],u=e[0],o=t[0]-i,a=r[0]-u,l=n[1],c=e[1],f=t[1]-l,s=r[1]-c,h=(a*(l-c)-s*(i-u))/(s*o-a*f);return[i+h*o,l+h*f]}function De(n){var t=n[0],e=n[n.length-1];return!(t[0]-e[0]||t[1]-e[1])}function Pe(){rr(this),this.edge=this.site=this.circle=null}function Ue(n){var t=cl.pop()||new Pe;return t.site=n,t}function je(n){Be(n),ol.remove(n),cl.push(n),rr(n)}function Fe(n){var t=n.circle,e=t.x,r=t.cy,i={x:e,y:r},u=n.P,o=n.N,a=[n];je(n);for(var l=u;l.circle&&xo(e-l.circle.x)<Uo&&xo(r-l.circle.cy)<Uo;)u=l.P,a.unshift(l),je(l),l=u;a.unshift(l),Be(l);for(var c=o;c.circle&&xo(e-c.circle.x)<Uo&&xo(r-c.circle.cy)<Uo;)o=c.N,a.push(c),je(c),c=o;a.push(c),Be(c);var f,s=a.length;for(f=1;s>f;++f)c=a[f],l=a[f-1],nr(c.edge,l.site,c.site,i);l=a[0],c=a[s-
1],c.edge=Ke(l.site,c.site,null,i),$e(l),$e(c)}function He(n){for(var t,e,r,i,u=n.x,o=n.y,a=ol._;a;)if(r=Oe(a,o)-u,r>Uo)a=a.L;else{if(i=u-Ie(a,o),!(i>Uo)){r>-Uo?(t=a.P,e=a):i>-Uo?(t=a,e=a.N):t=e=a;break}if(!a.R){t=a;break}a=a.R}var l=Ue(n);if(ol.insert(t,l),t||e){if(t===e)return Be(t),e=Ue(t.site),ol.insert(l,e),l.edge=e.edge=Ke(t.site,l.site),$e(t),void $e(e);if(!e)return void(l.edge=Ke(t.site,l.site));Be(t),Be(e);var c=t.site,f=c.x,s=c.y,h=n.x-f,p=n.y-s,g=e.site,v=g.x-f,d=g.y-s,y=2*(h*d-p*v),m=h*h+p*p,M=v*v+d*d,x={x:(d*m-p*M)/y+f,y:(h*M-v*m)/y+s};nr(e.edge,c,g,x),l.edge=Ke(c,n,null,x),e.edge=Ke(n,g,null,x),$e(t),$e(e)}}function Oe(n,t){var e=n.site,r=e.x,i=e.y,u=i-t;if(!u)return r;var o=n.P;if(!o)return-(1/0);e=o.site;var a=e.x,l=e.y,c=l-t;if(!c)return a;var f=a-r,s=1/u-1/c,h=f/c;return s?(-h+Math.sqrt(h*h-2*s*(f*f/(-2*c)-l+c/2+i-u/2)))/s+r:(r+a)/2}function Ie(n,t){var e=n.N;if(e)return Oe(e,t);var r=n.site;return r.y===t?r.x:1/0}function Ye(n){this.site=n,this.edges=[]}function Z
e(n){for(var t,e,r,i,u,o,a,l,c,f,s=n[0][0],h=n[1][0],p=n[0][1],g=n[1][1],v=ul,d=v.length;d--;)if(u=v[d],u&&u.prepare())for(a=u.edges,l=a.length,o=0;l>o;)f=a[o].end(),r=f.x,i=f.y,c=a[++o%l].start(),t=c.x,e=c.y,(xo(r-t)>Uo||xo(i-e)>Uo)&&(a.splice(o,0,new tr(Qe(u.site,f,xo(r-s)<Uo&&g-i>Uo?{x:s,y:xo(t-s)<Uo?e:g}:xo(i-g)<Uo&&h-r>Uo?{x:xo(e-g)<Uo?t:h,y:g}:xo(r-h)<Uo&&i-p>Uo?{x:h,y:xo(t-h)<Uo?e:p}:xo(i-p)<Uo&&r-s>Uo?{x:xo(e-p)<Uo?t:s,y:p}:null),u.site,null)),++l)}function Ve(n,t){return t.angle-n.angle}function Xe(){rr(this),this.x=this.y=this.arc=this.site=this.cy=null}function $e(n){var t=n.P,e=n.N;if(t&&e){var r=t.site,i=n.site,u=e.site;if(r!==u){var o=i.x,a=i.y,l=r.x-o,c=r.y-a,f=u.x-o,s=u.y-a,h=2*(l*s-c*f);if(!(h>=-jo)){var p=l*l+c*c,g=f*f+s*s,v=(s*p-c*g)/h,d=(l*g-f*p)/h,s=d+a,y=fl.pop()||new Xe;y.arc=n,y.site=i,y.x=v+o,y.y=s+Math.sqrt(v*v+d*d),y.cy=s,n.circle=y;for(var m=null,M=ll._;M;)if(y.y<M.y||y.y===M.y&&y.x<=M.x){if(!M.L){m=M.P;break}M=M.L}else{if(!M.R){m=M;break}M=M.R}ll.insert(
m,y),m||(al=y)}}}}function Be(n){var t=n.circle;t&&(t.P||(al=t.N),ll.remove(t),fl.push(t),rr(t),n.circle=null)}function We(n){for(var t,e=il,r=Yt(n[0][0],n[0][1],n[1][0],n[1][1]),i=e.length;i--;)t=e[i],(!Je(t,n)||!r(t)||xo(t.a.x-t.b.x)<Uo&&xo(t.a.y-t.b.y)<Uo)&&(t.a=t.b=null,e.splice(i,1))}function Je(n,t){var e=n.b;if(e)return!0;var r,i,u=n.a,o=t[0][0],a=t[1][0],l=t[0][1],c=t[1][1],f=n.l,s=n.r,h=f.x,p=f.y,g=s.x,v=s.y,d=(h+g)/2,y=(p+v)/2;if(v===p){if(o>d||d>=a)return;if(h>g){if(u){if(u.y>=c)return}else u={x:d,y:l};e={x:d,y:c}}else{if(u){if(u.y<l)return}else u={x:d,y:c};e={x:d,y:l}}}else if(r=(h-g)/(v-p),i=y-r*d,-1>r||r>1)if(h>g){if(u){if(u.y>=c)return}else u={x:(l-i)/r,y:l};e={x:(c-i)/r,y:c}}else{if(u){if(u.y<l)return}else u={x:(c-i)/r,y:c};e={x:(l-i)/r,y:l}}else if(v>p){if(u){if(u.x>=a)return}else u={x:o,y:r*o+i};e={x:a,y:r*a+i}}else{if(u){if(u.x<o)return}else u={x:a,y:r*a+i};e={x:o,y:r*o+i}}return n.a=u,n.b=e,!0}function Ge(n,t){this.l=n,this.r=t,this.a=this.b=null}function Ke(n,t,
e,r){var i=new Ge(n,t);return il.push(i),e&&nr(i,n,t,e),r&&nr(i,t,n,r),ul[n.i].edges.push(new tr(i,n,t)),ul[t.i].edges.push(new tr(i,t,n)),i}function Qe(n,t,e){var r=new Ge(n,null);return r.a=t,r.b=e,il.push(r),r}function nr(n,t,e,r){n.a||n.b?n.l===e?n.b=r:n.a=r:(n.a=r,n.l=t,n.r=e)}function tr(n,t,e){var r=n.a,i=n.b;this.edge=n,this.site=t,this.angle=e?Math.atan2(e.y-t.y,e.x-t.x):n.l===t?Math.atan2(i.x-r.x,r.y-i.y):Math.atan2(r.x-i.x,i.y-r.y)}function er(){this._=null}function rr(n){n.U=n.C=n.L=n.R=n.P=n.N=null}function ir(n,t){var e=t,r=t.R,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.R=r.L,e.R&&(e.R.U=e),r.L=e}function ur(n,t){var e=t,r=t.L,i=e.U;i?i.L===e?i.L=r:i.R=r:n._=r,r.U=i,e.U=r,e.L=r.R,e.L&&(e.L.U=e),r.R=e}function or(n){for(;n.L;)n=n.L;return n}function ar(n,t){var e,r,i,u=n.sort(lr).pop();for(il=[],ul=new Array(n.length),ol=new er,ll=new er;;)if(i=al,u&&(!i||u.y<i.y||u.y===i.y&&u.x<i.x))u.x===e&&u.y===r||(ul[u.i]=new Ye(u),He(u),e=u.x,r=u.y),u=n.pop();else{if(!i)break
;Fe(i.arc)}t&&(We(t),Ze(t));var o={cells:ul,edges:il};return ol=ll=il=ul=null,o}function lr(n,t){return t.y-n.y||t.x-n.x}function cr(n,t,e){return(n.x-e.x)*(t.y-n.y)-(n.x-t.x)*(e.y-n.y)}function fr(n){return n.x}function sr(n){return n.y}function hr(){return{leaf:!0,nodes:[],point:null,x:null,y:null}}function pr(n,t,e,r,i,u){if(!n(t,e,r,i,u)){var o=.5*(e+i),a=.5*(r+u),l=t.nodes;l[0]&&pr(n,l[0],e,r,o,a),l[1]&&pr(n,l[1],o,r,i,a),l[2]&&pr(n,l[2],e,a,o,u),l[3]&&pr(n,l[3],o,a,i,u)}}function gr(n,t,e,r,i,u,o){var a,l=1/0;return function c(n,f,s,h,p){if(!(f>u||s>o||r>h||i>p)){if(g=n.point){var g,v=t-n.x,d=e-n.y,y=v*v+d*d;if(l>y){var m=Math.sqrt(l=y);r=t-m,i=e-m,u=t+m,o=e+m,a=g}}for(var M=n.nodes,x=.5*(f+h),b=.5*(s+p),_=t>=x,w=e>=b,S=w<<1|_,k=S+4;k>S;++S)if(n=M[3&S])switch(3&S){case 0:c(n,f,s,x,b);break;case 1:c(n,x,s,h,b);break;case 2:c(n,f,b,x,p);break;case 3:c(n,x,b,h,p)}}}(n,r,i,u,o),a}function vr(n,t){n=ao.rgb(n),t=ao.rgb(t);var e=n.r,r=n.g,i=n.b,u=t.r-e,o=t.g-r,a=t.b-i;return function
(n){return"#"+bn(Math.round(e+u*n))+bn(Math.round(r+o*n))+bn(Math.round(i+a*n))}}function dr(n,t){var e,r={},i={};for(e in n)e in t?r[e]=Mr(n[e],t[e]):i[e]=n[e];for(e in t)e in n||(i[e]=t[e]);return function(n){for(e in r)i[e]=r[e](n);return i}}function yr(n,t){return n=+n,t=+t,function(e){return n*(1-e)+t*e}}function mr(n,t){var e,r,i,u=hl.lastIndex=pl.lastIndex=0,o=-1,a=[],l=[];for(n+="",t+="";(e=hl.exec(n))&&(r=pl.exec(t));)(i=r.index)>u&&(i=t.slice(u,i),a[o]?a[o]+=i:a[++o]=i),(e=e[0])===(r=r[0])?a[o]?a[o]+=r:a[++o]=r:(a[++o]=null,l.push({i:o,x:yr(e,r)})),u=pl.lastIndex;return u<t.length&&(i=t.slice(u),a[o]?a[o]+=i:a[++o]=i),a.length<2?l[0]?(t=l[0].x,function(n){return t(n)+""}):function(){return t}:(t=l.length,function(n){for(var e,r=0;t>r;++r)a[(e=l[r]).i]=e.x(n);return a.join("")})}function Mr(n,t){for(var e,r=ao.interpolators.length;--r>=0&&!(e=ao.interpolators[r](n,t)););return e}function xr(n,t){var e,r=[],i=[],u=n.length,o=t.length,a=Math.min(n.length,t.length);for(e=0;a>e
;++e)r.push(Mr(n[e],t[e]));for(;u>e;++e)i[e]=n[e];for(;o>e;++e)i[e]=t[e];return function(n){for(e=0;a>e;++e)i[e]=r[e](n);return i}}function br(n){return function(t){return 0>=t?0:t>=1?1:n(t)}}function _r(n){return function(t){return 1-n(1-t)}}function wr(n){return function(t){return.5*(.5>t?n(2*t):2-n(2-2*t))}}function Sr(n){return n*n}function kr(n){return n*n*n}function Nr(n){if(0>=n)return 0;if(n>=1)return 1;var t=n*n,e=t*n;return 4*(.5>n?e:3*(n-t)+e-.75)}function Er(n){return function(t){return Math.pow(t,n)}}function Ar(n){return 1-Math.cos(n*Io)}function Cr(n){return Math.pow(2,10*(n-1))}function zr(n){return 1-Math.sqrt(1-n*n)}function Lr(n,t){var e;return arguments.length<2&&(t=.45),arguments.length?e=t/Ho*Math.asin(1/n):(n=1,e=t/4),function(r){return 1+n*Math.pow(2,-10*r)*Math.sin((r-e)*Ho/t)}}function qr(n){return n||(n=1.70158),function(t){return t*t*((n+1)*t-n)}}function Tr(n){return 1/2.75>n?7.5625*n*n:2/2.75>n?7.5625*(n-=1.5/2.75)*n+.75:2.5/2.75>n?7.5625*(n-=2.25/2.75)
*n+.9375:7.5625*(n-=2.625/2.75)*n+.984375}function Rr(n,t){n=ao.hcl(n),t=ao.hcl(t);var e=n.h,r=n.c,i=n.l,u=t.h-e,o=t.c-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.c:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return sn(e+u*n,r+o*n,i+a*n)+""}}function Dr(n,t){n=ao.hsl(n),t=ao.hsl(t);var e=n.h,r=n.s,i=n.l,u=t.h-e,o=t.s-r,a=t.l-i;return isNaN(o)&&(o=0,r=isNaN(r)?t.s:r),isNaN(u)?(u=0,e=isNaN(e)?t.h:e):u>180?u-=360:-180>u&&(u+=360),function(n){return cn(e+u*n,r+o*n,i+a*n)+""}}function Pr(n,t){n=ao.lab(n),t=ao.lab(t);var e=n.l,r=n.a,i=n.b,u=t.l-e,o=t.a-r,a=t.b-i;return function(n){return pn(e+u*n,r+o*n,i+a*n)+""}}function Ur(n,t){return t-=n,function(e){return Math.round(n+t*e)}}function jr(n){var t=[n.a,n.b],e=[n.c,n.d],r=Hr(t),i=Fr(t,e),u=Hr(Or(e,t,-i))||0;t[0]*e[1]<e[0]*t[1]&&(t[0]*=-1,t[1]*=-1,r*=-1,i*=-1),this.rotate=(r?Math.atan2(t[1],t[0]):Math.atan2(-e[0],e[1]))*Zo,this.translate=[n.e,n.f],this.scale=[r,u],this.skew=u?Math.atan2(i,u)*Zo:0}function
Fr(n,t){return n[0]*t[0]+n[1]*t[1]}function Hr(n){var t=Math.sqrt(Fr(n,n));return t&&(n[0]/=t,n[1]/=t),t}function Or(n,t,e){return n[0]+=e*t[0],n[1]+=e*t[1],n}function Ir(n){return n.length?n.pop()+",":""}function Yr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push("translate(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else(t[0]||t[1])&&e.push("translate("+t+")")}function Zr(n,t,e,r){n!==t?(n-t>180?t+=360:t-n>180&&(n+=360),r.push({i:e.push(Ir(e)+"rotate(",null,")")-2,x:yr(n,t)})):t&&e.push(Ir(e)+"rotate("+t+")")}function Vr(n,t,e,r){n!==t?r.push({i:e.push(Ir(e)+"skewX(",null,")")-2,x:yr(n,t)}):t&&e.push(Ir(e)+"skewX("+t+")")}function Xr(n,t,e,r){if(n[0]!==t[0]||n[1]!==t[1]){var i=e.push(Ir(e)+"scale(",null,",",null,")");r.push({i:i-4,x:yr(n[0],t[0])},{i:i-2,x:yr(n[1],t[1])})}else 1===t[0]&&1===t[1]||e.push(Ir(e)+"scale("+t+")")}function $r(n,t){var e=[],r=[];return n=ao.transform(n),t=ao.transform(t),Yr(n.translate,t.translate,e,r),Zr(n.rotate,t.r
otate,e,r),Vr(n.skew,t.skew,e,r),Xr(n.scale,t.scale,e,r),n=t=null,function(n){for(var t,i=-1,u=r.length;++i<u;)e[(t=r[i]).i]=t.x(n);return e.join("")}}function Br(n,t){return t=(t-=n=+n)||1/t,function(e){return(e-n)/t}}function Wr(n,t){return t=(t-=n=+n)||1/t,function(e){return Math.max(0,Math.min(1,(e-n)/t))}}function Jr(n){for(var t=n.source,e=n.target,r=Kr(t,e),i=[t];t!==r;)t=t.parent,i.push(t);for(var u=i.length;e!==r;)i.splice(u,0,e),e=e.parent;return i}function Gr(n){for(var t=[],e=n.parent;null!=e;)t.push(n),n=e,e=e.parent;return t.push(n),t}function Kr(n,t){if(n===t)return n;for(var e=Gr(n),r=Gr(t),i=e.pop(),u=r.pop(),o=null;i===u;)o=i,i=e.pop(),u=r.pop();return o}function Qr(n){n.fixed|=2}function ni(n){n.fixed&=-7}function ti(n){n.fixed|=4,n.px=n.x,n.py=n.y}function ei(n){n.fixed&=-5}function ri(n,t,e){var r=0,i=0;if(n.charge=0,!n.leaf)for(var u,o=n.nodes,a=o.length,l=-1;++l<a;)u=o[l],null!=u&&(ri(u,t,e),n.charge+=u.charge,r+=u.charge*u.cx,i+=u.charge*u.cy);if(n.point){n.l
eaf||(n.point.x+=Math.random()-.5,n.point.y+=Math.random()-.5);var c=t*e[n.point.index];n.charge+=n.pointCharge=c,r+=c*n.point.x,i+=c*n.point.y}n.cx=r/n.charge,n.cy=i/n.charge}function ii(n,t){return ao.rebind(n,t,"sort","children","value"),n.nodes=n,n.links=fi,n}function ui(n,t){for(var e=[n];null!=(n=e.pop());)if(t(n),(i=n.children)&&(r=i.length))for(var r,i;--r>=0;)e.push(i[r])}function oi(n,t){for(var e=[n],r=[];null!=(n=e.pop());)if(r.push(n),(u=n.children)&&(i=u.length))for(var i,u,o=-1;++o<i;)e.push(u[o]);for(;null!=(n=r.pop());)t(n)}function ai(n){return n.children}function li(n){return n.value}function ci(n,t){return t.value-n.value}function fi(n){return ao.merge(n.map(function(n){return(n.children||[]).map(function(t){return{source:n,target:t}})}))}function si(n){return n.x}function hi(n){return n.y}function pi(n,t,e){n.y0=t,n.y=e}function gi(n){return ao.range(n.length)}function vi(n){for(var t=-1,e=n[0].length,r=[];++t<e;)r[t]=0;return r}function di(n){for(var t,e=1,r=0,
i=n[0][1],u=n.length;u>e;++e)(t=n[e][1])>i&&(r=e,i=t);return r}function yi(n){return n.reduce(mi,0)}function mi(n,t){return n+t[1]}function Mi(n,t){return xi(n,Math.ceil(Math.log(t.length)/Math.LN2+1))}function xi(n,t){for(var e=-1,r=+n[0],i=(n[1]-r)/t,u=[];++e<=t;)u[e]=i*e+r;return u}function bi(n){return[ao.min(n),ao.max(n)]}function _i(n,t){return n.value-t.value}function wi(n,t){var e=n._pack_next;n._pack_next=t,t._pack_prev=n,t._pack_next=e,e._pack_prev=t}function Si(n,t){n._pack_next=t,t._pack_prev=n}function ki(n,t){var e=t.x-n.x,r=t.y-n.y,i=n.r+t.r;return.999*i*i>e*e+r*r}function Ni(n){function t(n){f=Math.min(n.x-n.r,f),s=Math.max(n.x+n.r,s),h=Math.min(n.y-n.r,h),p=Math.max(n.y+n.r,p)}if((e=n.children)&&(c=e.length)){var e,r,i,u,o,a,l,c,f=1/0,s=-(1/0),h=1/0,p=-(1/0);if(e.forEach(Ei),r=e[0],r.x=-r.r,r.y=0,t(r),c>1&&(i=e[1],i.x=i.r,i.y=0,t(i),c>2))for(u=e[2],zi(r,i,u),t(u),wi(r,u),r._pack_prev=u,wi(u,i),i=r._pack_next,o=3;c>o;o++){zi(r,i,u=e[o]);var g=0,v=1,d=1;for(a=i._pack_
next;a!==i;a=a._pack_next,v++)if(ki(a,u)){g=1;break}if(1==g)for(l=r._pack_prev;l!==a._pack_prev&&!ki(l,u);l=l._pack_prev,d++);g?(d>v||v==d&&i.r<r.r?Si(r,i=a):Si(r=l,i),o--):(wi(r,u),i=u,t(u))}var y=(f+s)/2,m=(h+p)/2,M=0;for(o=0;c>o;o++)u=e[o],u.x-=y,u.y-=m,M=Math.max(M,u.r+Math.sqrt(u.x*u.x+u.y*u.y));n.r=M,e.forEach(Ai)}}function Ei(n){n._pack_next=n._pack_prev=n}function Ai(n){delete n._pack_next,delete n._pack_prev}function Ci(n,t,e,r){var i=n.children;if(n.x=t+=r*n.x,n.y=e+=r*n.y,n.r*=r,i)for(var u=-1,o=i.length;++u<o;)Ci(i[u],t,e,r)}function zi(n,t,e){var r=n.r+e.r,i=t.x-n.x,u=t.y-n.y;if(r&&(i||u)){var o=t.r+e.r,a=i*i+u*u;o*=o,r*=r;var l=.5+(r-o)/(2*a),c=Math.sqrt(Math.max(0,2*o*(r+a)-(r-=a)*r-o*o))/(2*a);e.x=n.x+l*i+c*u,e.y=n.y+l*u-c*i}else e.x=n.x+r,e.y=n.y}function Li(n,t){return n.parent==t.parent?1:2}function qi(n){var t=n.children;return t.length?t[0]:n.t}function Ti(n){var t,e=n.children;return(t=e.length)?e[t-1]:n.t}function Ri(n,t,e){var r=e/(t.i-n.i);t.c-=r,t.s+=e,n.c+
=r,t.z+=e,t.m+=e}function Di(n){for(var t,e=0,r=0,i=n.children,u=i.length;--u>=0;)t=i[u],t.z+=e,t.m+=e,e+=t.s+(r+=t.c)}function Pi(n,t,e){return n.a.parent===t.parent?n.a:e}function Ui(n){return 1+ao.max(n,function(n){return n.y})}function ji(n){return n.reduce(function(n,t){return n+t.x},0)/n.length}function Fi(n){var t=n.children;return t&&t.length?Fi(t[0]):n}function Hi(n){var t,e=n.children;return e&&(t=e.length)?Hi(e[t-1]):n}function Oi(n){return{x:n.x,y:n.y,dx:n.dx,dy:n.dy}}function Ii(n,t){var e=n.x+t[3],r=n.y+t[0],i=n.dx-t[1]-t[3],u=n.dy-t[0]-t[2];return 0>i&&(e+=i/2,i=0),0>u&&(r+=u/2,u=0),{x:e,y:r,dx:i,dy:u}}function Yi(n){var t=n[0],e=n[n.length-1];return e>t?[t,e]:[e,t]}function Zi(n){return n.rangeExtent?n.rangeExtent():Yi(n.range())}function Vi(n,t,e,r){var i=e(n[0],n[1]),u=r(t[0],t[1]);return function(n){return u(i(n))}}function Xi(n,t){var e,r=0,i=n.length-1,u=n[r],o=n[i];return u>o&&(e=r,r=i,i=e,e=u,u=o,o=e),n[r]=t.floor(u),n[i]=t.ceil(o),n}function $i(n){return n?{f
loor:function(t){return Math.floor(t/n)*n},ceil:function(t){return Math.ceil(t/n)*n}}:Sl}function Bi(n,t,e,r){var i=[],u=[],o=0,a=Math.min(n.length,t.length)-1;for(n[a]<n[0]&&(n=n.slice().reverse(),t=t.slice().reverse());++o<=a;)i.push(e(n[o-1],n[o])),u.push(r(t[o-1],t[o]));return function(t){var e=ao.bisect(n,t,1,a)-1;return u[e](i[e](t))}}function Wi(n,t,e,r){function i(){var i=Math.min(n.length,t.length)>2?Bi:Vi,l=r?Wr:Br;return o=i(n,t,l,e),a=i(t,n,l,Mr),u}function u(n){return o(n)}var o,a;return u.invert=function(n){return a(n)},u.domain=function(t){return arguments.length?(n=t.map(Number),i()):n},u.range=function(n){return arguments.length?(t=n,i()):t},u.rangeRound=function(n){return u.range(n).interpolate(Ur)},u.clamp=function(n){return arguments.length?(r=n,i()):r},u.interpolate=function(n){return arguments.length?(e=n,i()):e},u.ticks=function(t){return Qi(n,t)},u.tickFormat=function(t,e){return nu(n,t,e)},u.nice=function(t){return Gi(n,t),i()},u.copy=function(){return Wi(n,
t,e,r)},i()}function Ji(n,t){return ao.rebind(n,t,"range","rangeRound","interpolate","clamp")}function Gi(n,t){return Xi(n,$i(Ki(n,t)[2])),Xi(n,$i(Ki(n,t)[2])),n}function Ki(n,t){null==t&&(t=10);var e=Yi(n),r=e[1]-e[0],i=Math.pow(10,Math.floor(Math.log(r/t)/Math.LN10)),u=t/r*i;return.15>=u?i*=10:.35>=u?i*=5:.75>=u&&(i*=2),e[0]=Math.ceil(e[0]/i)*i,e[1]=Math.floor(e[1]/i)*i+.5*i,e[2]=i,e}function Qi(n,t){return ao.range.apply(ao,Ki(n,t))}function nu(n,t,e){var r=Ki(n,t);if(e){var i=ha.exec(e);if(i.shift(),"s"===i[8]){var u=ao.formatPrefix(Math.max(xo(r[0]),xo(r[1])));return i[7]||(i[7]="."+tu(u.scale(r[2]))),i[8]="f",e=ao.format(i.join("")),function(n){return e(u.scale(n))+u.symbol}}i[7]||(i[7]="."+eu(i[8],r)),e=i.join("")}else e=",."+tu(r[2])+"f";return ao.format(e)}function tu(n){return-Math.floor(Math.log(n)/Math.LN10+.01)}function eu(n,t){var e=tu(t[2]);return n in kl?Math.abs(e-tu(Math.max(xo(t[0]),xo(t[1]))))+ +("e"!==n):e-2*("%"===n)}function ru(n,t,e,r){function i(n){return(e?
Math.log(0>n?0:n):-Math.log(n>0?0:-n))/Math.log(t)}function u(n){return e?Math.pow(t,n):-Math.pow(t,-n)}function o(t){return n(i(t))}return o.invert=function(t){return u(n.invert(t))},o.domain=function(t){return arguments.length?(e=t[0]>=0,n.domain((r=t.map(Number)).map(i)),o):r},o.base=function(e){return arguments.length?(t=+e,n.domain(r.map(i)),o):t},o.nice=function(){var t=Xi(r.map(i),e?Math:El);return n.domain(t),r=t.map(u),o},o.ticks=function(){var n=Yi(r),o=[],a=n[0],l=n[1],c=Math.floor(i(a)),f=Math.ceil(i(l)),s=t%1?2:t;if(isFinite(f-c)){if(e){for(;f>c;c++)for(var h=1;s>h;h++)o.push(u(c)*h);o.push(u(c))}else for(o.push(u(c));c++<f;)for(var h=s-1;h>0;h--)o.push(u(c)*h);for(c=0;o[c]<a;c++);for(f=o.length;o[f-1]>l;f--);o=o.slice(c,f)}return o},o.tickFormat=function(n,e){if(!arguments.length)return Nl;arguments.length<2?e=Nl:"function"!=typeof e&&(e=ao.format(e));var r=Math.max(1,t*n/o.ticks().length);return function(n){var o=n/u(Math.round(i(n)));return t-.5>o*t&&(o*=t),r>=o?e(n)
:""}},o.copy=function(){return ru(n.copy(),t,e,r)},Ji(o,n)}function iu(n,t,e){function r(t){return n(i(t))}var i=uu(t),u=uu(1/t);return r.invert=function(t){return u(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain((e=t.map(Number)).map(i)),r):e},r.ticks=function(n){return Qi(e,n)},r.tickFormat=function(n,t){return nu(e,n,t)},r.nice=function(n){return r.domain(Gi(e,n))},r.exponent=function(o){return arguments.length?(i=uu(t=o),u=uu(1/t),n.domain(e.map(i)),r):t},r.copy=function(){return iu(n.copy(),t,e)},Ji(r,n)}function uu(n){return function(t){return 0>t?-Math.pow(-t,n):Math.pow(t,n)}}function ou(n,t){function e(e){return u[((i.get(e)||("range"===t.t?i.set(e,n.push(e)):NaN))-1)%u.length]}function r(t,e){return ao.range(n.length).map(function(n){return t+e*n})}var i,u,o;return e.domain=function(r){if(!arguments.length)return n;n=[],i=new c;for(var u,o=-1,a=r.length;++o<a;)i.has(u=r[o])||i.set(u,n.push(u));return e[t.t].apply(e,t.a)},e.range=function(n){return arg
uments.length?(u=n,o=0,t={t:"range",a:arguments},e):u},e.rangePoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=(l+c)/2,0):(c-l)/(n.length-1+a);return u=r(l+f*a/2,f),o=0,t={t:"rangePoints",a:arguments},e},e.rangeRoundPoints=function(i,a){arguments.length<2&&(a=0);var l=i[0],c=i[1],f=n.length<2?(l=c=Math.round((l+c)/2),0):(c-l)/(n.length-1+a)|0;return u=r(l+Math.round(f*a/2+(c-l-(n.length-1+a)*f)/2),f),o=0,t={t:"rangeRoundPoints",a:arguments},e},e.rangeBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=(s-f)/(n.length-a+2*l);return u=r(f+h*l,h),c&&u.reverse(),o=h*(1-a),t={t:"rangeBands",a:arguments},e},e.rangeRoundBands=function(i,a,l){arguments.length<2&&(a=0),arguments.length<3&&(l=a);var c=i[1]<i[0],f=i[c-0],s=i[1-c],h=Math.floor((s-f)/(n.length-a+2*l));return u=r(f+Math.round((s-f-(n.length-a)*h)/2),h),c&&u.reverse(),o=Math.round(h*(1-a)),t={t:"rangeRoundBands",a:arguments},e},e.rangeBand=fu
nction(){return o},e.rangeExtent=function(){return Yi(t.a[0])},e.copy=function(){return ou(n,t)},e.domain(n)}function au(n,t){function u(){var e=0,r=t.length;for(a=[];++e<r;)a[e-1]=ao.quantile(n,e/r);return o}function o(n){return isNaN(n=+n)?void 0:t[ao.bisect(a,n)]}var a;return o.domain=function(t){return arguments.length?(n=t.map(r).filter(i).sort(e),u()):n},o.range=function(n){return arguments.length?(t=n,u()):t},o.quantiles=function(){return a},o.invertExtent=function(e){return e=t.indexOf(e),0>e?[NaN,NaN]:[e>0?a[e-1]:n[0],e<a.length?a[e]:n[n.length-1]]},o.copy=function(){return au(n,t)},u()}function lu(n,t,e){function r(t){return e[Math.max(0,Math.min(o,Math.floor(u*(t-n))))]}function i(){return u=e.length/(t-n),o=e.length-1,r}var u,o;return r.domain=function(e){return arguments.length?(n=+e[0],t=+e[e.length-1],i()):[n,t]},r.range=function(n){return arguments.length?(e=n,i()):e},r.invertExtent=function(t){return t=e.indexOf(t),t=0>t?NaN:t/u+n,[t,t+1/u]},r.copy=function(){return
lu(n,t,e)},i()}function cu(n,t){function e(e){return e>=e?t[ao.bisect(n,e)]:void 0}return e.domain=function(t){return arguments.length?(n=t,e):n},e.range=function(n){return arguments.length?(t=n,e):t},e.invertExtent=function(e){return e=t.indexOf(e),[n[e-1],n[e]]},e.copy=function(){return cu(n,t)},e}function fu(n){function t(n){return+n}return t.invert=t,t.domain=t.range=function(e){return arguments.length?(n=e.map(t),t):n},t.ticks=function(t){return Qi(n,t)},t.tickFormat=function(t,e){return nu(n,t,e)},t.copy=function(){return fu(n)},t}function su(){return 0}function hu(n){return n.innerRadius}function pu(n){return n.outerRadius}function gu(n){return n.startAngle}function vu(n){return n.endAngle}function du(n){return n&&n.padAngle}function yu(n,t,e,r){return(n-e)*t-(t-r)*n>0?0:1}function mu(n,t,e,r,i){var u=n[0]-t[0],o=n[1]-t[1],a=(i?r:-r)/Math.sqrt(u*u+o*o),l=a*o,c=-a*u,f=n[0]+l,s=n[1]+c,h=t[0]+l,p=t[1]+c,g=(f+h)/2,v=(s+p)/2,d=h-f,y=p-s,m=d*d+y*y,M=e-r,x=f*p-h*s,b=(0>y?-1:1)*Math
.sqrt(Math.max(0,M*M*m-x*x)),_=(x*y-d*b)/m,w=(-x*d-y*b)/m,S=(x*y+d*b)/m,k=(-x*d+y*b)/m,N=_-g,E=w-v,A=S-g,C=k-v;return N*N+E*E>A*A+C*C&&(_=S,w=k),[[_-l,w-c],[_*e/M,w*e/M]]}function Mu(n){function t(t){function o(){c.push("M",u(n(f),a))}for(var l,c=[],f=[],s=-1,h=t.length,p=En(e),g=En(r);++s<h;)i.call(this,l=t[s],s)?f.push([+p.call(this,l,s),+g.call(this,l,s)]):f.length&&(o(),f=[]);return f.length&&o(),c.length?c.join(""):null}var e=Ce,r=ze,i=zt,u=xu,o=u.key,a=.7;return t.x=function(n){return arguments.length?(e=n,t):e},t.y=function(n){return arguments.length?(r=n,t):r},t.defined=function(n){return arguments.length?(i=n,t):i},t.interpolate=function(n){return arguments.length?(o="function"==typeof n?u=n:(u=Tl.get(n)||xu).key,t):o},t.tension=function(n){return arguments.length?(a=n,t):a},t}function xu(n){return n.length>1?n.join("L"):n+"Z"}function bu(n){return n.join("L")+"Z"}function _u(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r[0]+(r=n[t])[0])/2,"V",r[1])
;return e>1&&i.push("H",r[0]),i.join("")}function wu(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("V",(r=n[t])[1],"H",r[0]);return i.join("")}function Su(n){for(var t=0,e=n.length,r=n[0],i=[r[0],",",r[1]];++t<e;)i.push("H",(r=n[t])[0],"V",r[1]);return i.join("")}function ku(n,t){return n.length<4?xu(n):n[1]+Au(n.slice(1,-1),Cu(n,t))}function Nu(n,t){return n.length<3?bu(n):n[0]+Au((n.push(n[0]),n),Cu([n[n.length-2]].concat(n,[n[1]]),t))}function Eu(n,t){return n.length<3?xu(n):n[0]+Au(n,Cu(n,t))}function Au(n,t){if(t.length<1||n.length!=t.length&&n.length!=t.length+2)return xu(n);var e=n.length!=t.length,r="",i=n[0],u=n[1],o=t[0],a=o,l=1;if(e&&(r+="Q"+(u[0]-2*o[0]/3)+","+(u[1]-2*o[1]/3)+","+u[0]+","+u[1],i=n[1],l=2),t.length>1){a=t[1],u=n[l],l++,r+="C"+(i[0]+o[0])+","+(i[1]+o[1])+","+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1];for(var c=2;c<t.length;c++,l++)u=n[l],a=t[c],r+="S"+(u[0]-a[0])+","+(u[1]-a[1])+","+u[0]+","+u[1]}if(e){var f=n[l];r+="Q"+(u[0]+2*a[0]/3
)+","+(u[1]+2*a[1]/3)+","+f[0]+","+f[1]}return r}function Cu(n,t){for(var e,r=[],i=(1-t)/2,u=n[0],o=n[1],a=1,l=n.length;++a<l;)e=u,u=o,o=n[a],r.push([i*(o[0]-e[0]),i*(o[1]-e[1])]);return r}function zu(n){if(n.length<3)return xu(n);var t=1,e=n.length,r=n[0],i=r[0],u=r[1],o=[i,i,i,(r=n[1])[0]],a=[u,u,u,r[1]],l=[i,",",u,"L",Ru(Pl,o),",",Ru(Pl,a)];for(n.push(n[e-1]);++t<=e;)r=n[t],o.shift(),o.push(r[0]),a.shift(),a.push(r[1]),Du(l,o,a);return n.pop(),l.push("L",r),l.join("")}function Lu(n){if(n.length<4)return xu(n);for(var t,e=[],r=-1,i=n.length,u=[0],o=[0];++r<3;)t=n[r],u.push(t[0]),o.push(t[1]);for(e.push(Ru(Pl,u)+","+Ru(Pl,o)),--r;++r<i;)t=n[r],u.shift(),u.push(t[0]),o.shift(),o.push(t[1]),Du(e,u,o);return e.join("")}function qu(n){for(var t,e,r=-1,i=n.length,u=i+4,o=[],a=[];++r<4;)e=n[r%i],o.push(e[0]),a.push(e[1]);for(t=[Ru(Pl,o),",",Ru(Pl,a)],--r;++r<u;)e=n[r%i],o.shift(),o.push(e[0]),a.shift(),a.push(e[1]),Du(t,o,a);return t.join("")}function Tu(n,t){var e=n.length-1;if(e)for(va
r r,i,u=n[0][0],o=n[0][1],a=n[e][0]-u,l=n[e][1]-o,c=-1;++c<=e;)r=n[c],i=c/e,r[0]=t*r[0]+(1-t)*(u+i*a),r[1]=t*r[1]+(1-t)*(o+i*l);return zu(n)}function Ru(n,t){return n[0]*t[0]+n[1]*t[1]+n[2]*t[2]+n[3]*t[3]}function Du(n,t,e){n.push("C",Ru(Rl,t),",",Ru(Rl,e),",",Ru(Dl,t),",",Ru(Dl,e),",",Ru(Pl,t),",",Ru(Pl,e))}function Pu(n,t){return(t[1]-n[1])/(t[0]-n[0])}function Uu(n){for(var t=0,e=n.length-1,r=[],i=n[0],u=n[1],o=r[0]=Pu(i,u);++t<e;)r[t]=(o+(o=Pu(i=u,u=n[t+1])))/2;return r[t]=o,r}function ju(n){for(var t,e,r,i,u=[],o=Uu(n),a=-1,l=n.length-1;++a<l;)t=Pu(n[a],n[a+1]),xo(t)<Uo?o[a]=o[a+1]=0:(e=o[a]/t,r=o[a+1]/t,i=e*e+r*r,i>9&&(i=3*t/Math.sqrt(i),o[a]=i*e,o[a+1]=i*r));for(a=-1;++a<=l;)i=(n[Math.min(l,a+1)][0]-n[Math.max(0,a-1)][0])/(6*(1+o[a]*o[a])),u.push([i||0,o[a]*i||0]);return u}function Fu(n){return n.length<3?xu(n):n[0]+Au(n,ju(n))}function Hu(n){for(var t,e,r,i=-1,u=n.length;++i<u;)t=n[i],e=t[0],r=t[1]-Io,t[0]=e*Math.cos(r),t[1]=e*Math.sin(r);return n}function Ou(n){function t(t
){function l(){v.push("M",a(n(y),s),f,c(n(d.reverse()),s),"Z")}for(var h,p,g,v=[],d=[],y=[],m=-1,M=t.length,x=En(e),b=En(i),_=e===r?function(){
-return p}:En(r),w=i===u?function(){return g}:En(u);++m<M;)o.call(this,h=t[m],m)?(d.push([p=+x.call(this,h,m),g=+b.call(this,h,m)]),y.push([+_.call(this,h,m),+w.call(this,h,m)])):d.length&&(l(),d=[],y=[]);return d.length&&l(),v.length?v.join(""):null}var e=Ce,r=Ce,i=0,u=ze,o=zt,a=xu,l=a.key,c=a,f="L",s=.7;return t.x=function(n){return arguments.length?(e=r=n,t):r},t.x0=function(n){return arguments.length?(e=n,t):e},t.x1=function(n){return arguments.length?(r=n,t):r},t.y=function(n){return arguments.length?(i=u=n,t):u},t.y0=function(n){return arguments.length?(i=n,t):i},t.y1=function(n){return arguments.length?(u=n,t):u},t.defined=function(n){return arguments.length?(o=n,t):o},t.interpolate=function(n){return arguments.length?(l="function"==typeof n?a=n:(a=Tl.get(n)||xu).key,c=a.reverse||a,f=a.closed?"M":"L",t):l},t.tension=function(n){return arguments.length?(s=n,t):s},t}function Iu(n){return n.radius}function Yu(n){return[n.x,n.y]}function Zu(n){return function(){var t=n.apply(this,
arguments),e=t[0],r=t[1]-Io;return[e*Math.cos(r),e*Math.sin(r)]}}function Vu(){return 64}function Xu(){return"circle"}function $u(n){var t=Math.sqrt(n/Fo);return"M0,"+t+"A"+t+","+t+" 0 1,1 0,"+-t+"A"+t+","+t+" 0 1,1 0,"+t+"Z"}function Bu(n){return function(){var t,e,r;(t=this[n])&&(r=t[e=t.active])&&(r.timer.c=null,r.timer.t=NaN,--t.count?delete t[e]:delete this[n],t.active+=.5,r.event&&r.event.interrupt.call(this,this.__data__,r.index))}}function Wu(n,t,e){return ko(n,Yl),n.namespace=t,n.id=e,n}function Ju(n,t,e,r){var i=n.id,u=n.namespace;return Y(n,"function"==typeof e?function(n,o,a){n[u][i].tween.set(t,r(e.call(n,n.__data__,o,a)))}:(e=r(e),function(n){n[u][i].tween.set(t,e)}))}function Gu(n){return null==n&&(n=""),function(){this.textContent=n}}function Ku(n){return null==n?"__transition__":"__transition_"+n+"__"}function Qu(n,t,e,r,i){function u(n){var t=v.delay;return f.t=t+l,n>=t?o(n-t):void(f.c=o)}function o(e){var i=g.active,u=g[i];u&&(u.timer.c=null,u.timer.t=NaN,--g.coun
t,delete g[i],u.event&&u.event.interrupt.call(n,n.__data__,u.index));for(var o in g)if(r>+o){var c=g[o];c.timer.c=null,c.timer.t=NaN,--g.count,delete g[o]}f.c=a,qn(function(){return f.c&&a(e||1)&&(f.c=null,f.t=NaN),1},0,l),g.active=r,v.event&&v.event.start.call(n,n.__data__,t),p=[],v.tween.forEach(function(e,r){(r=r.call(n,n.__data__,t))&&p.push(r)}),h=v.ease,s=v.duration}function a(i){for(var u=i/s,o=h(u),a=p.length;a>0;)p[--a].call(n,o);return u>=1?(v.event&&v.event.end.call(n,n.__data__,t),--g.count?delete g[r]:delete n[e],1):void 0}var l,f,s,h,p,g=n[e]||(n[e]={active:0,count:0}),v=g[r];v||(l=i.time,f=qn(u,0,l),v=g[r]={tween:new c,time:l,timer:f,delay:i.delay,duration:i.duration,ease:i.ease,index:t},i=null,++g.count)}function no(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate("+(isFinite(r)?r:e(n))+",0)"})}function to(n,t,e){n.attr("transform",function(n){var r=t(n);return"translate(0,"+(isFinite(r)?r:e(n))+")"})}function eo(n){return n.toISOString()}function ro
(n,t,e){function r(t){return n(t)}function i(n,e){var r=n[1]-n[0],i=r/e,u=ao.bisect(Kl,i);return u==Kl.length?[t.year,Ki(n.map(function(n){return n/31536e6}),e)[2]]:u?t[i/Kl[u-1]<Kl[u]/i?u-1:u]:[tc,Ki(n,e)[2]]}return r.invert=function(t){return io(n.invert(t))},r.domain=function(t){return arguments.length?(n.domain(t),r):n.domain().map(io)},r.nice=function(n,t){function e(e){return!isNaN(e)&&!n.range(e,io(+e+1),t).length}var u=r.domain(),o=Yi(u),a=null==n?i(o,10):"number"==typeof n&&i(o,n);return a&&(n=a[0],t=a[1]),r.domain(Xi(u,t>1?{floor:function(t){for(;e(t=n.floor(t));)t=io(t-1);return t},ceil:function(t){for(;e(t=n.ceil(t));)t=io(+t+1);return t}}:n))},r.ticks=function(n,t){var e=Yi(r.domain()),u=null==n?i(e,10):"number"==typeof n?i(e,n):!n.range&&[{range:n},t];return u&&(n=u[0],t=u[1]),n.range(e[0],io(+e[1]+1),1>t?1:t)},r.tickFormat=function(){return e},r.copy=function(){return ro(n.copy(),t,e)},Ji(r,n)}function io(n){return new Date(n)}function uo(n){return JSON.parse(n.respon
seText)}function oo(n){var t=fo.createRange();return t.selectNode(fo.body),t.createContextualFragment(n.responseText)}var ao={version:"3.5.17"},lo=[].slice,co=function(n){return lo.call(n)},fo=this.document;if(fo)try{co(fo.documentElement.childNodes)[0].nodeType}catch(so){co=function(n){for(var t=n.length,e=new Array(t);t--;)e[t]=n[t];return e}}if(Date.now||(Date.now=function(){return+new Date}),fo)try{fo.createElement("DIV").style.setProperty("opacity",0,"")}catch(ho){var po=this.Element.prototype,go=po.setAttribute,vo=po.setAttributeNS,yo=this.CSSStyleDeclaration.prototype,mo=yo.setProperty;po.setAttribute=function(n,t){go.call(this,n,t+"")},po.setAttributeNS=function(n,t,e){vo.call(this,n,t,e+"")},yo.setProperty=function(n,t,e){mo.call(this,n,t+"",e)}}ao.ascending=e,ao.descending=function(n,t){return n>t?-1:t>n?1:t>=n?0:NaN},ao.min=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&e>r&&(e=r)}e
lse{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&e>r&&(e=r)}return e},ao.max=function(n,t){var e,r,i=-1,u=n.length;if(1===arguments.length){for(;++i<u;)if(null!=(r=n[i])&&r>=r){e=r;break}for(;++i<u;)null!=(r=n[i])&&r>e&&(e=r)}else{for(;++i<u;)if(null!=(r=t.call(n,n[i],i))&&r>=r){e=r;break}for(;++i<u;)null!=(r=t.call(n,n[i],i))&&r>e&&(e=r)}return e},ao.extent=function(n,t){var e,r,i,u=-1,o=n.length;if(1===arguments.length){for(;++u<o;)if(null!=(r=n[u])&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=n[u])&&(e>r&&(e=r),r>i&&(i=r))}else{for(;++u<o;)if(null!=(r=t.call(n,n[u],u))&&r>=r){e=i=r;break}for(;++u<o;)null!=(r=t.call(n,n[u],u))&&(e>r&&(e=r),r>i&&(i=r))}return[e,i]},ao.sum=function(n,t){var e,r=0,u=n.length,o=-1;if(1===arguments.length)for(;++o<u;)i(e=+n[o])&&(r+=e);else for(;++o<u;)i(e=+t.call(n,n[o],o))&&(r+=e);return r},ao.mean=function(n,t){var e,u=0,o=n.length,a=-1,l=o;if(1===arguments.length)for(;++a<o;)i(e=r(n[a]))?u+=e:--l;els
e for(;++a<o;)i(e=r(t.call(n,n[a],a)))?u+=e:--l;return l?u/l:void 0},ao.quantile=function(n,t){var e=(n.length-1)*t+1,r=Math.floor(e),i=+n[r-1],u=e-r;return u?i+u*(n[r]-i):i},ao.median=function(n,t){var u,o=[],a=n.length,l=-1;if(1===arguments.length)for(;++l<a;)i(u=r(n[l]))&&o.push(u);else for(;++l<a;)i(u=r(t.call(n,n[l],l)))&&o.push(u);return o.length?ao.quantile(o.sort(e),.5):void 0},ao.variance=function(n,t){var e,u,o=n.length,a=0,l=0,c=-1,f=0;if(1===arguments.length)for(;++c<o;)i(e=r(n[c]))&&(u=e-a,a+=u/++f,l+=u*(e-a));else for(;++c<o;)i(e=r(t.call(n,n[c],c)))&&(u=e-a,a+=u/++f,l+=u*(e-a));return f>1?l/(f-1):void 0},ao.deviation=function(){var n=ao.variance.apply(this,arguments);return n?Math.sqrt(n):n};var Mo=u(e);ao.bisectLeft=Mo.left,ao.bisect=ao.bisectRight=Mo.right,ao.bisector=function(n){return u(1===n.length?function(t,r){return e(n(t),r)}:n)},ao.shuffle=function(n,t,e){(u=arguments.length)<3&&(e=n.length,2>u&&(t=0));for(var r,i,u=e-t;u;)i=Math.random()*u--|0,r=n[u+t],n[u+
t]=n[i+t],n[i+t]=r;return n},ao.permute=function(n,t){for(var e=t.length,r=new Array(e);e--;)r[e]=n[t[e]];return r},ao.pairs=function(n){for(var t,e=0,r=n.length-1,i=n[0],u=new Array(0>r?0:r);r>e;)u[e]=[t=i,i=n[++e]];return u},ao.transpose=function(n){if(!(i=n.length))return[];for(var t=-1,e=ao.min(n,o),r=new Array(e);++t<e;)for(var i,u=-1,a=r[t]=new Array(i);++u<i;)a[u]=n[u][t];return r},ao.zip=function(){return ao.transpose(arguments)},ao.keys=function(n){var t=[];for(var e in n)t.push(e);return t},ao.values=function(n){var t=[];for(var e in n)t.push(n[e]);return t},ao.entries=function(n){var t=[];for(var e in n)t.push({key:e,value:n[e]});return t},ao.merge=function(n){for(var t,e,r,i=n.length,u=-1,o=0;++u<i;)o+=n[u].length;for(e=new Array(o);--i>=0;)for(r=n[i],t=r.length;--t>=0;)e[--o]=r[t];return e};var xo=Math.abs;ao.range=function(n,t,e){if(arguments.length<3&&(e=1,arguments.length<2&&(t=n,n=0)),(t-n)/e===1/0)throw new Error("infinite range");var r,i=[],u=a(xo(e)),o=-1;if(n*=u
,t*=u,e*=u,0>e)for(;(r=n+e*++o)>t;)i.push(r/u);else for(;(r=n+e*++o)<t;)i.push(r/u);return i},ao.map=function(n,t){var e=new c;if(n instanceof c)n.forEach(function(n,t){e.set(n,t)});else if(Array.isArray(n)){var r,i=-1,u=n.length;if(1===arguments.length)for(;++i<u;)e.set(i,n[i]);else for(;++i<u;)e.set(t.call(n,r=n[i],i),r)}else for(var o in n)e.set(o,n[o]);return e};var bo="__proto__",_o="\x00";l(c,{has:h,get:function(n){return this._[f(n)]},set:function(n,t){return this._[f(n)]=t},remove:p,keys:g,values:function(){var n=[];for(var t in this._)n.push(this._[t]);return n},entries:function(){var n=[];for(var t in this._)n.push({key:s(t),value:this._[t]});return n},size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t),this._[t])}}),ao.nest=function(){function n(t,o,a){if(a>=u.length)return r?r.call(i,o):e?o.sort(e):o;for(var l,f,s,h,p=-1,g=o.length,v=u[a++],d=new c;++p<g;)(h=d.get(l=v(f=o[p])))?h.push(f):d.set(l,[f]);return t?(f=t(),s=function(e,r){f.set(e,n(t,r,a))})
:(f={},s=function(e,r){f[e]=n(t,r,a)}),d.forEach(s),f}function t(n,e){if(e>=u.length)return n;var r=[],i=o[e++];return n.forEach(function(n,i){r.push({key:n,values:t(i,e)})}),i?r.sort(function(n,t){return i(n.key,t.key)}):r}var e,r,i={},u=[],o=[];return i.map=function(t,e){return n(e,t,0)},i.entries=function(e){return t(n(ao.map,e,0),0)},i.key=function(n){return u.push(n),i},i.sortKeys=function(n){return o[u.length-1]=n,i},i.sortValues=function(n){return e=n,i},i.rollup=function(n){return r=n,i},i},ao.set=function(n){var t=new y;if(n)for(var e=0,r=n.length;r>e;++e)t.add(n[e]);return t},l(y,{has:h,add:function(n){return this._[f(n+="")]=!0,n},remove:p,values:g,size:v,empty:d,forEach:function(n){for(var t in this._)n.call(this,s(t))}}),ao.behavior={},ao.rebind=function(n,t){for(var e,r=1,i=arguments.length;++r<i;)n[e=arguments[r]]=M(n,t,t[e]);return n};var wo=["webkit","ms","moz","Moz","o","O"];ao.dispatch=function(){for(var n=new _,t=-1,e=arguments.length;++t<e;)n[arguments[t]]=w(n);
return n},_.prototype.on=function(n,t){var e=n.indexOf("."),r="";if(e>=0&&(r=n.slice(e+1),n=n.slice(0,e)),n)return arguments.length<2?this[n].on(r):this[n].on(r,t);if(2===arguments.length){if(null==t)for(n in this)this.hasOwnProperty(n)&&this[n].on(r,null);return this}},ao.event=null,ao.requote=function(n){return n.replace(So,"\\$&")};var So=/[\\\^\$\*\+\?\|\[\]\(\)\.\{\}]/g,ko={}.__proto__?function(n,t){n.__proto__=t}:function(n,t){for(var e in t)n[e]=t[e]},No=function(n,t){return t.querySelector(n)},Eo=function(n,t){return t.querySelectorAll(n)},Ao=function(n,t){var e=n.matches||n[x(n,"matchesSelector")];return(Ao=function(n,t){return e.call(n,t)})(n,t)};"function"==typeof Sizzle&&(No=function(n,t){return Sizzle(n,t)[0]||null},Eo=Sizzle,Ao=Sizzle.matchesSelector),ao.selection=function(){return ao.select(fo.documentElement)};var Co=ao.selection.prototype=[];Co.select=function(n){var t,e,r,i,u=[];n=A(n);for(var o=-1,a=this.length;++o<a;){u.push(t=[]),t.parentNode=(r=this[o]).parentN
ode;for(var l=-1,c=r.length;++l<c;)(i=r[l])?(t.push(e=n.call(i,i.__data__,l,o)),e&&"__data__"in i&&(e.__data__=i.__data__)):t.push(null)}return E(u)},Co.selectAll=function(n){var t,e,r=[];n=C(n);for(var i=-1,u=this.length;++i<u;)for(var o=this[i],a=-1,l=o.length;++a<l;)(e=o[a])&&(r.push(t=co(n.call(e,e.__data__,a,i))),t.parentNode=e);return E(r)};var zo="http://www.w3.org/1999/xhtml",Lo={svg:"http://www.w3.org/2000/svg",xhtml:zo,xlink:"http://www.w3.org/1999/xlink",xml:"http://www.w3.org/XML/1998/namespace",xmlns:"http://www.w3.org/2000/xmlns/"};ao.ns={prefix:Lo,qualify:function(n){var t=n.indexOf(":"),e=n;return t>=0&&"xmlns"!==(e=n.slice(0,t))&&(n=n.slice(t+1)),Lo.hasOwnProperty(e)?{space:Lo[e],local:n}:n}},Co.attr=function(n,t){if(arguments.length<2){if("string"==typeof n){var e=this.node();return n=ao.ns.qualify(n),n.local?e.getAttributeNS(n.space,n.local):e.getAttribute(n)}for(t in n)this.each(z(t,n[t]));return this}return this.each(z(n,t))},Co.classed=function(n,t){if(argument
s.length<2){if("string"==typeof n){var e=this.node(),r=(n=T(n)).length,i=-1;if(t=e.classList){for(;++i<r;)if(!t.contains(n[i]))return!1}else for(t=e.getAttribute("class");++i<r;)if(!q(n[i]).test(t))return!1;return!0}for(t in n)this.each(R(t,n[t]));return this}return this.each(R(n,t))},Co.style=function(n,e,r){var i=arguments.length;if(3>i){if("string"!=typeof n){2>i&&(e="");for(r in n)this.each(P(r,n[r],e));return this}if(2>i){var u=this.node();return t(u).getComputedStyle(u,null).getPropertyValue(n)}r=""}return this.each(P(n,e,r))},Co.property=function(n,t){if(arguments.length<2){if("string"==typeof n)return this.node()[n];for(t in n)this.each(U(t,n[t]));return this}return this.each(U(n,t))},Co.text=function(n){return arguments.length?this.each("function"==typeof n?function(){var t=n.apply(this,arguments);this.textContent=null==t?"":t}:null==n?function(){this.textContent=""}:function(){this.textContent=n}):this.node().textContent},Co.html=function(n){return arguments.length?this.ea
ch("function"==typeof n?function(){var t=n.apply(this,arguments);this.innerHTML=null==t?"":t}:null==n?function(){this.innerHTML=""}:function(){this.innerHTML=n}):this.node().innerHTML},Co.append=function(n){return n=j(n),this.select(function(){return this.appendChild(n.apply(this,arguments))})},Co.insert=function(n,t){return n=j(n),t=A(t),this.select(function(){return this.insertBefore(n.apply(this,arguments),t.apply(this,arguments)||null)})},Co.remove=function(){return this.each(F)},Co.data=function(n,t){function e(n,e){var r,i,u,o=n.length,s=e.length,h=Math.min(o,s),p=new Array(s),g=new Array(s),v=new Array(o);if(t){var d,y=new c,m=new Array(o);for(r=-1;++r<o;)(i=n[r])&&(y.has(d=t.call(i,i.__data__,r))?v[r]=i:y.set(d,i),m[r]=d);for(r=-1;++r<s;)(i=y.get(d=t.call(e,u=e[r],r)))?i!==!0&&(p[r]=i,i.__data__=u):g[r]=H(u),y.set(d,!0);for(r=-1;++r<o;)r in m&&y.get(m[r])!==!0&&(v[r]=n[r])}else{for(r=-1;++r<h;)i=n[r],u=e[r],i?(i.__data__=u,p[r]=i):g[r]=H(u);for(;s>r;++r)g[r]=H(e[r]);for(;o>r
;++r)v[r]=n[r]}g.update=p,g.parentNode=p.parentNode=v.parentNode=n.parentNode,a.push(g),l.push(p),f.push(v)}var r,i,u=-1,o=this.length;if(!arguments.length){for(n=new Array(o=(r=this[0]).length);++u<o;)(i=r[u])&&(n[u]=i.__data__);return n}var a=Z([]),l=E([]),f=E([]);if("function"==typeof n)for(;++u<o;)e(r=this[u],n.call(r,r.parentNode.__data__,u));else for(;++u<o;)e(r=this[u],n);return l.enter=function(){return a},l.exit=function(){return f},l},Co.datum=function(n){return arguments.length?this.property("__data__",n):this.property("__data__")},Co.filter=function(n){var t,e,r,i=[];"function"!=typeof n&&(n=O(n));for(var u=0,o=this.length;o>u;u++){i.push(t=[]),t.parentNode=(e=this[u]).parentNode;for(var a=0,l=e.length;l>a;a++)(r=e[a])&&n.call(r,r.__data__,a,u)&&t.push(r)}return E(i)},Co.order=function(){for(var n=-1,t=this.length;++n<t;)for(var e,r=this[n],i=r.length-1,u=r[i];--i>=0;)(e=r[i])&&(u&&u!==e.nextSibling&&u.parentNode.insertBefore(e,u),u=e);return this},Co.sort=function(n){n=
I.apply(this,arguments);for(var t=-1,e=this.length;++t<e;)this[t].sort(n);return this.order()},Co.each=function(n){return Y(this,function(t,e,r){n.call(t,t.__data__,e,r)})},Co.call=function(n){var t=co(arguments);return n.apply(t[0]=this,t),this},Co.empty=function(){return!this.node()},Co.node=function(){for(var n=0,t=this.length;t>n;n++)for(var e=this[n],r=0,i=e.length;i>r;r++){var u=e[r];if(u)return u}return null},Co.size=function(){var n=0;return Y(this,function(){++n}),n};var qo=[];ao.selection.enter=Z,ao.selection.enter.prototype=qo,qo.append=Co.append,qo.empty=Co.empty,qo.node=Co.node,qo.call=Co.call,qo.size=Co.size,qo.select=function(n){for(var t,e,r,i,u,o=[],a=-1,l=this.length;++a<l;){r=(i=this[a]).update,o.push(t=[]),t.parentNode=i.parentNode;for(var c=-1,f=i.length;++c<f;)(u=i[c])?(t.push(r[c]=e=n.call(i.parentNode,u.__data__,c,a)),e.__data__=u.__data__):t.push(null)}return E(o)},qo.insert=function(n,t){return arguments.length<2&&(t=V(this)),Co.insert.call(this,n,t)},ao.se
lect=function(t){var e;return"string"==typeof t?(e=[No(t,fo)],e.parentNode=fo.documentElement):(e=[t],e.parentNode=n(t)),E([e])},ao.selectAll=function(n){var t;return"string"==typeof n?(t=co(Eo(n,fo)),t.parentNode=fo.documentElement):(t=co(n),t.parentNode=null),E([t])},Co.on=function(n,t,e){var r=arguments.length;if(3>r){if("string"!=typeof n){2>r&&(t=!1);for(e in n)this.each(X(e,n[e],t));return this}if(2>r)return(r=this.node()["__on"+n])&&r._;e=!1}return this.each(X(n,t,e))};var To=ao.map({mouseenter:"mouseover",mouseleave:"mouseout"});fo&&To.forEach(function(n){"on"+n in fo&&To.remove(n)});var Ro,Do=0;ao.mouse=function(n){return J(n,k())};var Po=this.navigator&&/WebKit/.test(this.navigator.userAgent)?-1:0;ao.touch=function(n,t,e){if(arguments.length<3&&(e=t,t=k().changedTouches),t)for(var r,i=0,u=t.length;u>i;++i)if((r=t[i]).identifier===e)return J(n,r)},ao.behavior.drag=function(){function n(){this.on("mousedown.drag",u).on("touchstart.drag",o)}function e(n,t,e,u,o){return functi
on(){function a(){var n,e,r=t(h,v);r&&(n=r[0]-M[0],e=r[1]-M[1],g|=n|e,M=r,p({type:"drag",x:r[0]+c[0],y:r[1]+c[1],dx:n,dy:e}))}function l(){t(h,v)&&(y.on(u+d,null).on(o+d,null),m(g),p({type:"dragend"}))}var c,f=this,s=ao.event.target.correspondingElement||ao.event.target,h=f.parentNode,p=r.of(f,arguments),g=0,v=n(),d=".drag"+(null==v?"":"-"+v),y=ao.select(e(s)).on(u+d,a).on(o+d,l),m=W(s),M=t(h,v);i?(c=i.apply(f,arguments),c=[c.x-M[0],c.y-M[1]]):c=[0,0],p({type:"dragstart"})}}var r=N(n,"drag","dragstart","dragend"),i=null,u=e(b,ao.mouse,t,"mousemove","mouseup"),o=e(G,ao.touch,m,"touchmove","touchend");return n.origin=function(t){return arguments.length?(i=t,n):i},ao.rebind(n,r,"on")},ao.touches=function(n,t){return arguments.length<2&&(t=k().touches),t?co(t).map(function(t){var e=J(n,t);return e.identifier=t.identifier,e}):[]};var Uo=1e-6,jo=Uo*Uo,Fo=Math.PI,Ho=2*Fo,Oo=Ho-Uo,Io=Fo/2,Yo=Fo/180,Zo=180/Fo,Vo=Math.SQRT2,Xo=2,$o=4;ao.interpolateZoom=function(n,t){var e,r,i=n[0],u=n[1],o=n[
2],a=t[0],l=t[1],c=t[2],f=a-i,s=l-u,h=f*f+s*s;if(jo>h)r=Math.log(c/o)/Vo,e=function(n){return[i+n*f,u+n*s,o*Math.exp(Vo*n*r)]};else{var p=Math.sqrt(h),g=(c*c-o*o+$o*h)/(2*o*Xo*p),v=(c*c-o*o-$o*h)/(2*c*Xo*p),d=Math.log(Math.sqrt(g*g+1)-g),y=Math.log(Math.sqrt(v*v+1)-v);r=(y-d)/Vo,e=function(n){var t=n*r,e=rn(d),a=o/(Xo*p)*(e*un(Vo*t+d)-en(d));return[i+a*f,u+a*s,o*e/rn(Vo*t+d)]}}return e.duration=1e3*r,e},ao.behavior.zoom=function(){function n(n){n.on(L,s).on(Wo+".zoom",p).on("dblclick.zoom",g).on(R,h)}function e(n){return[(n[0]-k.x)/k.k,(n[1]-k.y)/k.k]}function r(n){return[n[0]*k.k+k.x,n[1]*k.k+k.y]}function i(n){k.k=Math.max(A[0],Math.min(A[1],n))}function u(n,t){t=r(t),k.x+=n[0]-t[0],k.y+=n[1]-t[1]}function o(t,e,r,o){t.__chart__={x:k.x,y:k.y,k:k.k},i(Math.pow(2,o)),u(d=e,r),t=ao.select(t),C>0&&(t=t.transition().duration(C)),t.call(n.event)}function a(){b&&b.domain(x.range().map(function(n){return(n-k.x)/k.k}).map(x.invert)),w&&w.domain(_.range().map(function(n){return(n-k.y)/k.k})
.map(_.invert))}function l(n){z++||n({type:"zoomstart"})}function c(n){a(),n({type:"zoom",scale:k.k,translate:[k.x,k.y]})}function f(n){--z||(n({type:"zoomend"}),d=null)}function s(){function n(){a=1,u(ao.mouse(i),h),c(o)}function r(){s.on(q,null).on(T,null),p(a),f(o)}var i=this,o=D.of(i,arguments),a=0,s=ao.select(t(i)).on(q,n).on(T,r),h=e(ao.mouse(i)),p=W(i);Il.call(i),l(o)}function h(){function n(){var n=ao.touches(g);return p=k.k,n.forEach(function(n){n.identifier in d&&(d[n.identifier]=e(n))}),n}function t(){var t=ao.event.target;ao.select(t).on(x,r).on(b,a),_.push(t);for(var e=ao.event.changedTouches,i=0,u=e.length;u>i;++i)d[e[i].identifier]=null;var l=n(),c=Date.now();if(1===l.length){if(500>c-M){var f=l[0];o(g,f,d[f.identifier],Math.floor(Math.log(k.k)/Math.LN2)+1),S()}M=c}else if(l.length>1){var f=l[0],s=l[1],h=f[0]-s[0],p=f[1]-s[1];y=h*h+p*p}}function r(){var n,t,e,r,o=ao.touches(g);Il.call(g);for(var a=0,l=o.length;l>a;++a,r=null)if(e=o[a],r=d[e.identifier]){if(t)break;n=e
,t=r}if(r){var f=(f=e[0]-n[0])*f+(f=e[1]-n[1])*f,s=y&&Math.sqrt(f/y);n=[(n[0]+e[0])/2,(n[1]+e[1])/2],t=[(t[0]+r[0])/2,(t[1]+r[1])/2],i(s*p)}M=null,u(n,t),c(v)}function a(){if(ao.event.touches.length){for(var t=ao.event.changedTouches,e=0,r=t.length;r>e;++e)delete d[t[e].identifier];for(var i in d)return void n()}ao.selectAll(_).on(m,null),w.on(L,s).on(R,h),N(),f(v)}var p,g=this,v=D.of(g,arguments),d={},y=0,m=".zoom-"+ao.event.changedTouches[0].identifier,x="touchmove"+m,b="touchend"+m,_=[],w=ao.select(g),N=W(g);t(),l(v),w.on(L,null).on(R,t)}function p(){var n=D.of(this,arguments);m?clearTimeout(m):(Il.call(this),v=e(d=y||ao.mouse(this)),l(n)),m=setTimeout(function(){m=null,f(n)},50),S(),i(Math.pow(2,.002*Bo())*k.k),u(d,v),c(n)}function g(){var n=ao.mouse(this),t=Math.log(k.k)/Math.LN2;o(this,n,e(n),ao.event.shiftKey?Math.ceil(t)-1:Math.floor(t)+1)}var v,d,y,m,M,x,b,_,w,k={x:0,y:0,k:1},E=[960,500],A=Jo,C=250,z=0,L="mousedown.zoom",q="mousemove.zoom",T="mouseup.zoom",R="touchstart.zoo
m",D=N(n,"zoomstart","zoom","zoomend");return Wo||(Wo="onwheel"in fo?(Bo=function(){return-ao.event.deltaY*(ao.event.deltaMode?120:1)},"wheel"):"onmousewheel"in fo?(Bo=function(){return ao.event.wheelDelta},"mousewheel"):(Bo=function(){return-ao.event.detail},"MozMousePixelScroll")),n.event=function(n){n.each(function(){var n=D.of(this,arguments),t=k;Hl?ao.select(this).transition().each("start.zoom",function(){k=this.__chart__||{x:0,y:0,k:1},l(n)}).tween("zoom:zoom",function(){var e=E[0],r=E[1],i=d?d[0]:e/2,u=d?d[1]:r/2,o=ao.interpolateZoom([(i-k.x)/k.k,(u-k.y)/k.k,e/k.k],[(i-t.x)/t.k,(u-t.y)/t.k,e/t.k]);return function(t){var r=o(t),a=e/r[2];this.__chart__=k={x:i-r[0]*a,y:u-r[1]*a,k:a},c(n)}}).each("interrupt.zoom",function(){f(n)}).each("end.zoom",function(){f(n)}):(this.__chart__=k,l(n),c(n),f(n))})},n.translate=function(t){return arguments.length?(k={x:+t[0],y:+t[1],k:k.k},a(),n):[k.x,k.y]},n.scale=function(t){return arguments.length?(k={x:k.x,y:k.y,k:null},i(+t),a(),n):k.k},n.s
caleExtent=function(t){return arguments.length?(A=null==t?Jo:[+t[0],+t[1]],n):A},n.center=function(t){return arguments.length?(y=t&&[+t[0],+t[1]],n):y},n.size=function(t){return arguments.length?(E=t&&[+t[0],+t[1]],n):E},n.duration=function(t){return arguments.length?(C=+t,n):C},n.x=function(t){return arguments.length?(b=t,x=t.copy(),k={x:0,y:0,k:1},n):b},n.y=function(t){return arguments.length?(w=t,_=t.copy(),k={x:0,y:0,k:1},n):w},ao.rebind(n,D,"on")};var Bo,Wo,Jo=[0,1/0];ao.color=an,an.prototype.toString=function(){return this.rgb()+""},ao.hsl=ln;var Go=ln.prototype=new an;Go.brighter=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,this.l/n)},Go.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new ln(this.h,this.s,n*this.l)},Go.rgb=function(){return cn(this.h,this.s,this.l)},ao.hcl=fn;var Ko=fn.prototype=new an;Ko.brighter=function(n){return new fn(this.h,this.c,Math.min(100,this.l+Qo*(arguments.length?n:1)))},Ko.darker=function(n){return n
ew fn(this.h,this.c,Math.max(0,this.l-Qo*(arguments.length?n:1)))},Ko.rgb=function(){return sn(this.h,this.c,this.l).rgb()},ao.lab=hn;var Qo=18,na=.95047,ta=1,ea=1.08883,ra=hn.prototype=new an;ra.brighter=function(n){return new hn(Math.min(100,this.l+Qo*(arguments.length?n:1)),this.a,this.b)},ra.darker=function(n){return new hn(Math.max(0,this.l-Qo*(arguments.length?n:1)),this.a,this.b)},ra.rgb=function(){return pn(this.l,this.a,this.b)},ao.rgb=mn;var ia=mn.prototype=new an;ia.brighter=function(n){n=Math.pow(.7,arguments.length?n:1);var t=this.r,e=this.g,r=this.b,i=30;return t||e||r?(t&&i>t&&(t=i),e&&i>e&&(e=i),r&&i>r&&(r=i),new mn(Math.min(255,t/n),Math.min(255,e/n),Math.min(255,r/n))):new mn(i,i,i)},ia.darker=function(n){return n=Math.pow(.7,arguments.length?n:1),new mn(n*this.r,n*this.g,n*this.b)},ia.hsl=function(){return wn(this.r,this.g,this.b)},ia.toString=function(){return"#"+bn(this.r)+bn(this.g)+bn(this.b)};var ua=ao.map({aliceblue:15792383,antiquewhite:16444375,aqua:65535,
aquamarine:8388564,azure:15794175,beige:16119260,bisque:16770244,black:0,blanchedalmond:16772045,blue:255,blueviolet:9055202,brown:10824234,burlywood:14596231,cadetblue:6266528,chartreuse:8388352,chocolate:13789470,coral:16744272,cornflowerblue:6591981,cornsilk:16775388,crimson:14423100,cyan:65535,darkblue:139,darkcyan:35723,darkgoldenrod:12092939,darkgray:11119017,darkgreen:25600,darkgrey:11119017,darkkhaki:12433259,darkmagenta:9109643,darkolivegreen:5597999,darkorange:16747520,darkorchid:10040012,darkred:9109504,darksalmon:15308410,darkseagreen:9419919,darkslateblue:4734347,darkslategray:3100495,darkslategrey:3100495,darkturquoise:52945,darkviolet:9699539,deeppink:16716947,deepskyblue:49151,dimgray:6908265,dimgrey:6908265,dodgerblue:2003199,firebrick:11674146,floralwhite:16775920,forestgreen:2263842,fuchsia:16711935,gainsboro:14474460,ghostwhite:16316671,gold:16766720,goldenrod:14329120,gray:8421504,green:32768,greenyellow:11403055,grey:8421504,honeydew:15794
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
deleted file mode 100644
index 994b245..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/StateContext.java
+++ /dev/null
@@ -1,281 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.states.datanode.InitDatanodeState;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.datanode.RunningDatanodeState;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.Queue;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-import static org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ReportState.states
- .noContainerReports;
-
-/**
- * Current Context of State Machine.
- */
-public class StateContext {
- static final Logger LOG =
- LoggerFactory.getLogger(StateContext.class);
- private final Queue<SCMCommand> commandQueue;
- private final Lock lock;
- private final DatanodeStateMachine parent;
- private final AtomicLong stateExecutionCount;
- private final Configuration conf;
- private DatanodeStateMachine.DatanodeStates state;
- private SCMNodeReport nrState;
- private ReportState reportState;
- private static final ReportState DEFAULT_REPORT_STATE =
- ReportState.newBuilder().setState(noContainerReports).setCount(0).build();
-
- /**
- * Constructs a StateContext.
- *
- * @param conf - Configration
- * @param state - State
- * @param parent Parent State Machine
- */
- public StateContext(Configuration conf, DatanodeStateMachine.DatanodeStates
- state, DatanodeStateMachine parent) {
- this.conf = conf;
- this.state = state;
- this.parent = parent;
- commandQueue = new LinkedList<>();
- lock = new ReentrantLock();
- stateExecutionCount = new AtomicLong(0);
- nrState = SCMNodeReport.getDefaultInstance();
- }
-
- /**
- * Returns the ContainerStateMachine class that holds this state.
- *
- * @return ContainerStateMachine.
- */
- public DatanodeStateMachine getParent() {
- return parent;
- }
-
- /**
- * Get the container server port.
- * @return The container server port if available, return -1 if otherwise
- */
- public int getContainerPort() {
- return parent == null ?
- INVALID_PORT : parent.getContainer().getContainerServerPort();
- }
-
- /**
- * Gets the Ratis Port.
- * @return int , return -1 if not valid.
- */
- public int getRatisPort() {
- return parent == null ?
- INVALID_PORT : parent.getContainer().getRatisContainerServerPort();
- }
-
- /**
- * Returns true if we are entering a new state.
- *
- * @return boolean
- */
- boolean isEntering() {
- return stateExecutionCount.get() == 0;
- }
-
- /**
- * Returns true if we are exiting from the current state.
- *
- * @param newState - newState.
- * @return boolean
- */
- boolean isExiting(DatanodeStateMachine.DatanodeStates newState) {
- boolean isExiting = state != newState && stateExecutionCount.get() > 0;
- if(isExiting) {
- stateExecutionCount.set(0);
- }
- return isExiting;
- }
-
- /**
- * Returns the current state the machine is in.
- *
- * @return state.
- */
- public DatanodeStateMachine.DatanodeStates getState() {
- return state;
- }
-
- /**
- * Sets the current state of the machine.
- *
- * @param state state.
- */
- public void setState(DatanodeStateMachine.DatanodeStates state) {
- this.state = state;
- }
-
- /**
- * Returns the node report of the datanode state context.
- * @return the node report.
- */
- public SCMNodeReport getNodeReport() {
- return nrState;
- }
-
- /**
- * Sets the storage location report of the datanode state context.
- * @param nrReport - node report
- */
- public void setReportState(SCMNodeReport nrReport) {
- this.nrState = nrReport;
- }
-
- /**
- * Returns the next task to get executed by the datanode state machine.
- * @return A callable that will be executed by the
- * {@link DatanodeStateMachine}
- */
- @SuppressWarnings("unchecked")
- public DatanodeState<DatanodeStateMachine.DatanodeStates> getTask() {
- switch (this.state) {
- case INIT:
- return new InitDatanodeState(this.conf, parent.getConnectionManager(),
- this);
- case RUNNING:
- return new RunningDatanodeState(this.conf, parent.getConnectionManager(),
- this);
- case SHUTDOWN:
- return null;
- default:
- throw new IllegalArgumentException("Not Implemented yet.");
- }
- }
-
- /**
- * Executes the required state function.
- *
- * @param service - Executor Service
- * @param time - seconds to wait
- * @param unit - Seconds.
- * @throws InterruptedException
- * @throws ExecutionException
- * @throws TimeoutException
- */
- public void execute(ExecutorService service, long time, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- stateExecutionCount.incrementAndGet();
- DatanodeState<DatanodeStateMachine.DatanodeStates> task = getTask();
- if (this.isEntering()) {
- task.onEnter();
- }
- task.execute(service);
- DatanodeStateMachine.DatanodeStates newState = task.await(time, unit);
- if (this.state != newState) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Task {} executed, state transited from {} to {}",
- task.getClass().getSimpleName(), this.state, newState);
- }
- if (isExiting(newState)) {
- task.onExit();
- }
- this.clearReportState();
- this.setState(newState);
- }
- }
-
- /**
- * Returns the next command or null if it is empty.
- *
- * @return SCMCommand or Null.
- */
- public SCMCommand getNextCommand() {
- lock.lock();
- try {
- return commandQueue.poll();
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Adds a command to the State Machine queue.
- *
- * @param command - SCMCommand.
- */
- public void addCommand(SCMCommand command) {
- lock.lock();
- try {
- commandQueue.add(command);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Returns the count of the Execution.
- * @return long
- */
- public long getExecutionCount() {
- return stateExecutionCount.get();
- }
-
-
- /**
- * Gets the ReportState.
- * @return ReportState.
- */
- public synchronized ReportState getContainerReportState() {
- if (reportState == null) {
- return DEFAULT_REPORT_STATE;
- }
- return reportState;
- }
-
- /**
- * Sets the ReportState.
- * @param rState - ReportState.
- */
- public synchronized void setContainerReportState(ReportState rState) {
- this.reportState = rState;
- }
-
- /**
- * Clears report state after it has been communicated.
- */
- public synchronized void clearReportState() {
- if(reportState != null) {
- setContainerReportState(null);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
deleted file mode 100644
index 80844cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/BlockDeletingService.java
+++ /dev/null
@@ -1,237 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.background;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.util.List;
-import java.util.LinkedList;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
-
-/**
- * A per-datanode container block deleting service takes in charge
- * of deleting staled ozone blocks.
- */
-public class BlockDeletingService extends BackgroundService{
-
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockDeletingService.class);
-
- private final ContainerManager containerManager;
- private final Configuration conf;
-
- // Throttle number of blocks to delete per task,
- // set to 1 for testing
- private final int blockLimitPerTask;
-
- // Throttle the number of containers to process concurrently at a time,
- private final int containerLimitPerInterval;
-
- // Task priority is useful when a to-delete block has weight.
- private final static int TASK_PRIORITY_DEFAULT = 1;
- // Core pool size for container tasks
- private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 10;
-
- public BlockDeletingService(ContainerManager containerManager,
- long serviceInterval, long serviceTimeout, Configuration conf) {
- super("BlockDeletingService", serviceInterval,
- TimeUnit.MILLISECONDS, BLOCK_DELETING_SERVICE_CORE_POOL_SIZE,
- serviceTimeout);
- this.containerManager = containerManager;
- this.conf = conf;
- this.blockLimitPerTask = conf.getInt(
- OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER,
- OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT);
- this.containerLimitPerInterval = conf.getInt(
- OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL,
- OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT);
- }
-
- @Override
- public BackgroundTaskQueue getTasks() {
- BackgroundTaskQueue queue = new BackgroundTaskQueue();
- List<ContainerData> containers = Lists.newArrayList();
- try {
- // We at most list a number of containers a time,
- // in case there are too many containers and start too many workers.
- // We must ensure there is no empty container in this result.
- // The chosen result depends on what container deletion policy is
- // configured.
- containers = containerManager.chooseContainerForBlockDeletion(
- containerLimitPerInterval);
- LOG.info("Plan to choose {} containers for block deletion, "
- + "actually returns {} valid containers.",
- containerLimitPerInterval, containers.size());
-
- for(ContainerData container : containers) {
- BlockDeletingTask containerTask =
- new BlockDeletingTask(container, TASK_PRIORITY_DEFAULT);
- queue.add(containerTask);
- }
- } catch (StorageContainerException e) {
- LOG.warn("Failed to initiate block deleting tasks, "
- + "caused by unable to get containers info. "
- + "Retry in next interval. ", e);
- } catch (Exception e) {
- // In case listContainer call throws any uncaught RuntimeException.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unexpected error occurs during deleting blocks.", e);
- }
- }
- return queue;
- }
-
- private static class ContainerBackgroundTaskResult
- implements BackgroundTaskResult {
- private List<String> deletedBlockIds;
-
- ContainerBackgroundTaskResult() {
- deletedBlockIds = new LinkedList<>();
- }
-
- public void addBlockId(String blockId) {
- deletedBlockIds.add(blockId);
- }
-
- public void addAll(List<String> blockIds) {
- deletedBlockIds.addAll(blockIds);
- }
-
- public List<String> getDeletedBlocks() {
- return deletedBlockIds;
- }
-
- @Override
- public int getSize() {
- return deletedBlockIds.size();
- }
- }
-
- private class BlockDeletingTask
- implements BackgroundTask<BackgroundTaskResult> {
-
- private final int priority;
- private final ContainerData containerData;
-
- BlockDeletingTask(ContainerData containerName, int priority) {
- this.priority = priority;
- this.containerData = containerName;
- }
-
- @Override
- public BackgroundTaskResult call() throws Exception {
- ContainerBackgroundTaskResult crr = new ContainerBackgroundTaskResult();
- long startTime = Time.monotonicNow();
- // Scan container's db and get list of under deletion blocks
- MetadataStore meta = KeyUtils.getDB(containerData, conf);
- // # of blocks to delete is throttled
- KeyPrefixFilter filter = new KeyPrefixFilter(
- OzoneConsts.DELETING_KEY_PREFIX);
- List<Map.Entry<byte[], byte[]>> toDeleteBlocks =
- meta.getSequentialRangeKVs(null, blockLimitPerTask, filter);
- if (toDeleteBlocks.isEmpty()) {
- LOG.debug("No under deletion block found in container : {}",
- containerData.getContainerName());
- }
-
- List<String> succeedBlocks = new LinkedList<>();
- LOG.debug("Container : {}, To-Delete blocks : {}",
- containerData.getContainerName(), toDeleteBlocks.size());
- File dataDir = ContainerUtils.getDataDirectory(containerData).toFile();
- if (!dataDir.exists() || !dataDir.isDirectory()) {
- LOG.error("Invalid container data dir {} : "
- + "not exist or not a directory", dataDir.getAbsolutePath());
- return crr;
- }
-
- toDeleteBlocks.forEach(entry -> {
- String blockName = DFSUtil.bytes2String(entry.getKey());
- LOG.debug("Deleting block {}", blockName);
- try {
- ContainerProtos.KeyData data =
- ContainerProtos.KeyData.parseFrom(entry.getValue());
- for (ContainerProtos.ChunkInfo chunkInfo : data.getChunksList()) {
- File chunkFile = dataDir.toPath()
- .resolve(chunkInfo.getChunkName()).toFile();
- if (FileUtils.deleteQuietly(chunkFile)) {
- LOG.debug("block {} chunk {} deleted", blockName,
- chunkFile.getAbsolutePath());
- }
- }
- succeedBlocks.add(blockName);
- } catch (InvalidProtocolBufferException e) {
- LOG.error("Failed to parse block info for block {}", blockName, e);
- }
- });
-
- // Once files are deleted ... clean up DB
- BatchOperation batch = new BatchOperation();
- succeedBlocks.forEach(entry ->
- batch.delete(DFSUtil.string2Bytes(entry)));
- meta.writeBatch(batch);
- // update count of pending deletion blocks in in-memory container status
- containerManager.decrPendingDeletionBlocks(succeedBlocks.size(),
- containerData.getContainerName());
-
- if (!succeedBlocks.isEmpty()) {
- LOG.info("Container: {}, deleted blocks: {}, task elapsed time: {}ms",
- containerData.getContainerName(), succeedBlocks.size(),
- Time.monotonicNow() - startTime);
- }
- crr.addAll(succeedBlocks);
- return crr;
- }
-
- @Override
- public int getPriority() {
- return priority;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
deleted file mode 100644
index a9e202e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/background/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.background;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
deleted file mode 100644
index f5fbcd7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CloseContainerHandler.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.container.common.statemachine
- .SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCloseContainerCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Container Report handler.
- */
-public class CloseContainerHandler implements CommandHandler {
- static final Logger LOG =
- LoggerFactory.getLogger(CloseContainerHandler.class);
- private int invocationCount;
- private long totalTime;
-
- /**
- * Constructs a ContainerReport handler.
- */
- public CloseContainerHandler() {
- }
-
- /**
- * Handles a given SCM command.
- *
- * @param command - SCM Command
- * @param container - Ozone Container.
- * @param context - Current Context.
- * @param connectionManager - The SCMs that we are talking to.
- */
- @Override
- public void handle(SCMCommand command, OzoneContainer container,
- StateContext context, SCMConnectionManager connectionManager) {
- LOG.debug("Processing Close Container command.");
- invocationCount++;
- long startTime = Time.monotonicNow();
- String containerName = "UNKNOWN";
- try {
-
- SCMCloseContainerCmdResponseProto
- closeContainerProto =
- SCMCloseContainerCmdResponseProto
- .parseFrom(command.getProtoBufMessage());
- containerName = closeContainerProto.getContainerName();
-
- container.getContainerManager().closeContainer(containerName);
-
- } catch (Exception e) {
- LOG.error("Can't close container " + containerName, e);
- } finally {
- long endTime = Time.monotonicNow();
- totalTime += endTime - startTime;
- }
- }
-
- /**
- * Returns the command type that this command handler handles.
- *
- * @return Type
- */
- @Override
- public Type getCommandType() {
- return Type.closeContainerCommand;
- }
-
- /**
- * Returns number of times this handler has been invoked.
- *
- * @return int
- */
- @Override
- public int getInvocationCount() {
- return invocationCount;
- }
-
- /**
- * Returns the average time this function takes to run.
- *
- * @return long
- */
- @Override
- public long getAverageRunTime() {
- if (invocationCount > 0) {
- return totalTime / invocationCount;
- }
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
deleted file mode 100644
index 6670ea9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandDispatcher.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Dispatches command to the correct handler.
- */
-public final class CommandDispatcher {
- static final Logger LOG =
- LoggerFactory.getLogger(CommandDispatcher.class);
- private final StateContext context;
- private final Map<Type, CommandHandler> handlerMap;
- private final OzoneContainer container;
- private final SCMConnectionManager connectionManager;
-
- /**
- * Constructs a command Dispatcher.
- * @param context - Context.
- */
- /**
- * Constructs a command dispatcher.
- *
- * @param container - Ozone Container
- * @param context - Context
- * @param handlers - Set of handlers.
- */
- private CommandDispatcher(OzoneContainer container, SCMConnectionManager
- connectionManager, StateContext context,
- CommandHandler... handlers) {
- Preconditions.checkNotNull(context);
- Preconditions.checkNotNull(handlers);
- Preconditions.checkArgument(handlers.length > 0);
- Preconditions.checkNotNull(container);
- Preconditions.checkNotNull(connectionManager);
- this.context = context;
- this.container = container;
- this.connectionManager = connectionManager;
- handlerMap = new HashMap<>();
- for (CommandHandler h : handlers) {
- if(handlerMap.containsKey(h.getCommandType())){
- LOG.error("Duplicate handler for the same command. Exiting. Handle " +
- "key : { }", h.getCommandType().getDescriptorForType().getName());
- throw new IllegalArgumentException("Duplicate handler for the same " +
- "command.");
- }
- handlerMap.put(h.getCommandType(), h);
- }
- }
-
- /**
- * Dispatch the command to the correct handler.
- *
- * @param command - SCM Command.
- */
- public void handle(SCMCommand command) {
- Preconditions.checkNotNull(command);
- CommandHandler handler = handlerMap.get(command.getType());
- if (handler != null) {
- handler.handle(command, container, context, connectionManager);
- } else {
- LOG.error("Unknown SCM Command queued. There is no handler for this " +
- "command. Command: {}", command.getType().getDescriptorForType()
- .getName());
- }
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Helper class to construct command dispatcher.
- */
- public static class Builder {
- private final List<CommandHandler> handlerList;
- private OzoneContainer container;
- private StateContext context;
- private SCMConnectionManager connectionManager;
-
- public Builder() {
- handlerList = new LinkedList<>();
- }
-
- /**
- * Adds a handler.
- *
- * @param handler - handler
- * @return Builder
- */
- public Builder addHandler(CommandHandler handler) {
- Preconditions.checkNotNull(handler);
- handlerList.add(handler);
- return this;
- }
-
- /**
- * Add the OzoneContainer.
- *
- * @param ozoneContainer - ozone container.
- * @return Builder
- */
- public Builder setContainer(OzoneContainer ozoneContainer) {
- Preconditions.checkNotNull(ozoneContainer);
- this.container = ozoneContainer;
- return this;
- }
-
- /**
- * Set the Connection Manager.
- *
- * @param scmConnectionManager
- * @return this
- */
- public Builder setConnectionManager(SCMConnectionManager
- scmConnectionManager) {
- Preconditions.checkNotNull(scmConnectionManager);
- this.connectionManager = scmConnectionManager;
- return this;
- }
-
- /**
- * Sets the Context.
- *
- * @param stateContext - StateContext
- * @return this
- */
- public Builder setContext(StateContext stateContext) {
- Preconditions.checkNotNull(stateContext);
- this.context = stateContext;
- return this;
- }
-
- /**
- * Builds a command Dispatcher.
- * @return Command Dispatcher.
- */
- public CommandDispatcher build() {
- Preconditions.checkNotNull(this.connectionManager, "Missing connection" +
- " manager.");
- Preconditions.checkNotNull(this.container, "Missing container.");
- Preconditions.checkNotNull(this.context, "Missing context.");
- Preconditions.checkArgument(this.handlerList.size() > 0);
- return new CommandDispatcher(this.container, this.connectionManager,
- this.context, handlerList.toArray(
- new CommandHandler[handlerList.size()]));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
deleted file mode 100644
index b30d481..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/CommandHandler.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-
-/**
- * Generic interface for handlers.
- */
-public interface CommandHandler {
-
- /**
- * Handles a given SCM command.
- * @param command - SCM Command
- * @param container - Ozone Container.
- * @param context - Current Context.
- * @param connectionManager - The SCMs that we are talking to.
- */
- void handle(SCMCommand command, OzoneContainer container,
- StateContext context, SCMConnectionManager connectionManager);
-
- /**
- * Returns the command type that this command handler handles.
- * @return Type
- */
- Type getCommandType();
-
- /**
- * Returns number of times this handler has been invoked.
- * @return int
- */
- int getInvocationCount();
-
- /**
- * Returns the average time this function takes to run.
- * @return long
- */
- long getAverageRunTime();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
deleted file mode 100644
index 57a1516..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/ContainerReportHandler.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-
-/**
- * Container Report handler.
- */
-public class ContainerReportHandler implements CommandHandler {
- static final Logger LOG =
- LoggerFactory.getLogger(ContainerReportHandler.class);
- private int invocationCount;
- private long totalTime;
-
- /**
- * Constructs a ContainerReport handler.
- */
- public ContainerReportHandler() {
- }
-
- /**
- * Handles a given SCM command.
- *
- * @param command - SCM Command
- * @param container - Ozone Container.
- * @param context - Current Context.
- * @param connectionManager - The SCMs that we are talking to.
- */
- @Override
- public void handle(SCMCommand command, OzoneContainer container,
- StateContext context, SCMConnectionManager connectionManager) {
- LOG.debug("Processing Container Report.");
- invocationCount++;
- long startTime = Time.monotonicNow();
- try {
- ContainerReportsRequestProto contianerReport =
- container.getContainerReport();
-
- // TODO : We send this report to all SCMs.Check if it is enough only to
- // send to the leader once we have RAFT enabled SCMs.
- for (EndpointStateMachine endPoint : connectionManager.getValues()) {
- endPoint.getEndPoint().sendContainerReport(contianerReport);
- }
- } catch (IOException ex) {
- LOG.error("Unable to process the Container Report command.", ex);
- } finally {
- long endTime = Time.monotonicNow();
- totalTime += endTime - startTime;
- }
- }
-
- /**
- * Returns the command type that this command handler handles.
- *
- * @return Type
- */
- @Override
- public Type getCommandType() {
- return Type.sendContainerReport;
- }
-
- /**
- * Returns number of times this handler has been invoked.
- *
- * @return int
- */
- @Override
- public int getInvocationCount() {
- return invocationCount;
- }
-
- /**
- * Returns the average time this function takes to run.
- *
- * @return long
- */
- @Override
- public long getAverageRunTime() {
- if (invocationCount > 0) {
- return totalTime / invocationCount;
- }
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
deleted file mode 100644
index 1c859ea..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
-
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.DeletedContainerBlocksSummary;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Handle block deletion commands.
- */
-public class DeleteBlocksCommandHandler implements CommandHandler {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(DeleteBlocksCommandHandler.class);
-
- private ContainerManager containerManager;
- private Configuration conf;
- private int invocationCount;
- private long totalTime;
-
- public DeleteBlocksCommandHandler(ContainerManager containerManager,
- Configuration conf) {
- this.containerManager = containerManager;
- this.conf = conf;
- }
-
- @Override
- public void handle(SCMCommand command, OzoneContainer container,
- StateContext context, SCMConnectionManager connectionManager) {
- if (command.getType() != Type.deleteBlocksCommand) {
- LOG.warn("Skipping handling command, expected command "
- + "type {} but found {}",
- Type.deleteBlocksCommand, command.getType());
- return;
- }
- LOG.debug("Processing block deletion command.");
- invocationCount++;
- long startTime = Time.monotonicNow();
-
- // move blocks to deleting state.
- // this is a metadata update, the actual deletion happens in another
- // recycling thread.
- DeleteBlocksCommand cmd = (DeleteBlocksCommand) command;
- List<DeletedBlocksTransaction> containerBlocks = cmd.blocksTobeDeleted();
-
-
- DeletedContainerBlocksSummary summary =
- DeletedContainerBlocksSummary.getFrom(containerBlocks);
- LOG.info("Start to delete container blocks, TXIDs={}, "
- + "numOfContainers={}, numOfBlocks={}",
- summary.getTxIDSummary(),
- summary.getNumOfContainers(),
- summary.getNumOfBlocks());
-
- ContainerBlocksDeletionACKProto.Builder resultBuilder =
- ContainerBlocksDeletionACKProto.newBuilder();
- containerBlocks.forEach(entry -> {
- DeleteBlockTransactionResult.Builder txResultBuilder =
- DeleteBlockTransactionResult.newBuilder();
- txResultBuilder.setTxID(entry.getTxID());
- try {
- deleteContainerBlocks(entry, conf);
- txResultBuilder.setSuccess(true);
- } catch (IOException e) {
- LOG.warn("Failed to delete blocks for container={}, TXID={}",
- entry.getContainerName(), entry.getTxID(), e);
- txResultBuilder.setSuccess(false);
- }
- resultBuilder.addResults(txResultBuilder.build());
- });
- ContainerBlocksDeletionACKProto blockDeletionACK = resultBuilder.build();
-
- // Send ACK back to SCM as long as meta updated
- // TODO Or we should wait until the blocks are actually deleted?
- if (!containerBlocks.isEmpty()) {
- for (EndpointStateMachine endPoint : connectionManager.getValues()) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sending following block deletion ACK to SCM");
- for (DeleteBlockTransactionResult result :
- blockDeletionACK.getResultsList()) {
- LOG.debug(result.getTxID() + " : " + result.getSuccess());
- }
- }
- endPoint.getEndPoint()
- .sendContainerBlocksDeletionACK(blockDeletionACK);
- } catch (IOException e) {
- LOG.error("Unable to send block deletion ACK to SCM {}",
- endPoint.getAddress().toString(), e);
- }
- }
- }
-
- long endTime = Time.monotonicNow();
- totalTime += endTime - startTime;
- }
-
- /**
- * Move a bunch of blocks from a container to deleting state.
- * This is a meta update, the actual deletes happen in async mode.
- *
- * @param delTX a block deletion transaction.
- * @param config configuration.
- * @throws IOException if I/O error occurs.
- */
- private void deleteContainerBlocks(DeletedBlocksTransaction delTX,
- Configuration config) throws IOException {
- String containerId = delTX.getContainerName();
- ContainerData containerInfo = containerManager.readContainer(containerId);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing Container : {}, DB path : {}", containerId,
- containerInfo.getDBPath());
- }
-
- int newDeletionBlocks = 0;
- MetadataStore containerDB = KeyUtils.getDB(containerInfo, config);
- for (String blk : delTX.getBlockIDList()) {
- BatchOperation batch = new BatchOperation();
- byte[] blkBytes = DFSUtil.string2Bytes(blk);
- byte[] blkInfo = containerDB.get(blkBytes);
- if (blkInfo != null) {
- // Found the block in container db,
- // use an atomic update to change its state to deleting.
- batch.put(DFSUtil.string2Bytes(OzoneConsts.DELETING_KEY_PREFIX + blk),
- blkInfo);
- batch.delete(blkBytes);
- try {
- containerDB.writeBatch(batch);
- newDeletionBlocks++;
- LOG.debug("Transited Block {} to DELETING state in container {}",
- blk, containerId);
- } catch (IOException e) {
- // if some blocks failed to delete, we fail this TX,
- // without sending this ACK to SCM, SCM will resend the TX
- // with a certain number of retries.
- throw new IOException(
- "Failed to delete blocks for TXID = " + delTX.getTxID(), e);
- }
- } else {
- LOG.debug("Block {} not found or already under deletion in"
- + " container {}, skip deleting it.", blk, containerId);
- }
- }
-
- // update pending deletion blocks count in in-memory container status
- containerManager.incrPendingDeletionBlocks(newDeletionBlocks, containerId);
- }
-
- @Override
- public Type getCommandType() {
- return Type.deleteBlocksCommand;
- }
-
- @Override
- public int getInvocationCount() {
- return this.invocationCount;
- }
-
- @Override
- public long getAverageRunTime() {
- if (invocationCount > 0) {
- return totalTime / invocationCount;
- }
- return 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
deleted file mode 100644
index 1e9c8dc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine.commandhandler;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
deleted file mode 100644
index feb2f81..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-/**
-
- State machine class is used by the container to denote various states a
- container can be in and also is used for command processing.
-
- Container has the following states.
-
- Start - > getVersion -> Register -> Running -> Shutdown
-
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
deleted file mode 100644
index 75142af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/DatanodeState.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.states;
-
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * State Interface that allows tasks to maintain states.
- */
-public interface DatanodeState<T> {
- /**
- * Called before entering this state.
- */
- void onEnter();
-
- /**
- * Called After exiting this state.
- */
- void onExit();
-
- /**
- * Executes one or more tasks that is needed by this state.
- *
- * @param executor - ExecutorService
- */
- void execute(ExecutorService executor);
-
- /**
- * Wait for execute to finish.
- *
- * @param time - Time
- * @param timeUnit - Unit of time.
- */
- T await(long time, TimeUnit timeUnit)
- throws InterruptedException, ExecutionException, TimeoutException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
deleted file mode 100644
index 6ca94bf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/InitDatanodeState.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.Collection;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Init Datanode State is the task that gets run when we are in Init State.
- */
-public class InitDatanodeState implements DatanodeState,
- Callable<DatanodeStateMachine.DatanodeStates> {
- static final Logger LOG = LoggerFactory.getLogger(InitDatanodeState.class);
- private final SCMConnectionManager connectionManager;
- private final Configuration conf;
- private final StateContext context;
- private Future<DatanodeStateMachine.DatanodeStates> result;
-
- /**
- * Create InitDatanodeState Task.
- *
- * @param conf - Conf
- * @param connectionManager - Connection Manager
- * @param context - Current Context
- */
- public InitDatanodeState(Configuration conf,
- SCMConnectionManager connectionManager,
- StateContext context) {
- this.conf = conf;
- this.connectionManager = connectionManager;
- this.context = context;
- }
-
- /**
- * Computes a result, or throws an exception if unable to do so.
- *
- * @return computed result
- * @throws Exception if unable to compute a result
- */
- @Override
- public DatanodeStateMachine.DatanodeStates call() throws Exception {
- Collection<InetSocketAddress> addresses = null;
- try {
- addresses = OzoneClientUtils.getSCMAddresses(conf);
- } catch (IllegalArgumentException e) {
- if(!Strings.isNullOrEmpty(e.getMessage())) {
- LOG.error("Failed to get SCM addresses: " + e.getMessage());
- }
- return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
- }
-
- if (addresses == null || addresses.isEmpty()) {
- LOG.error("Null or empty SCM address list found.");
- return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
- } else {
- for (InetSocketAddress addr : addresses) {
- connectionManager.addSCMServer(addr);
- }
- }
-
- // If datanode ID is set, persist it to the ID file.
- persistContainerDatanodeID();
-
- return this.context.getState().getNextState();
- }
-
- /**
- * Update Ozone container port to the datanode ID,
- * and persist the ID to a local file.
- */
- private void persistContainerDatanodeID() throws IOException {
- String dataNodeIDPath = OzoneUtils.getDatanodeIDPath(conf);
- if (Strings.isNullOrEmpty(dataNodeIDPath)) {
- LOG.error("A valid file path is needed for config setting {}",
- ScmConfigKeys.OZONE_SCM_DATANODE_ID);
- this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
- return;
- }
- File idPath = new File(dataNodeIDPath);
- int containerPort = this.context.getContainerPort();
- int ratisPort = this.context.getRatisPort();
- DatanodeID datanodeID = this.context.getParent().getDatanodeID();
- if (datanodeID != null) {
- datanodeID.setContainerPort(containerPort);
- datanodeID.setRatisPort(ratisPort);
- ContainerUtils.writeDatanodeIDTo(datanodeID, idPath);
- LOG.info("Datanode ID is persisted to {}", dataNodeIDPath);
- }
- }
-
- /**
- * Called before entering this state.
- */
- @Override
- public void onEnter() {
- LOG.trace("Entering init container state");
- }
-
- /**
- * Called After exiting this state.
- */
- @Override
- public void onExit() {
- LOG.trace("Exiting init container state");
- }
-
- /**
- * Executes one or more tasks that is needed by this state.
- *
- * @param executor - ExecutorService
- */
- @Override
- public void execute(ExecutorService executor) {
- result = executor.submit(this);
- }
-
- /**
- * Wait for execute to finish.
- *
- * @param time - Time
- * @param timeUnit - Unit of time.
- */
- @Override
- public DatanodeStateMachine.DatanodeStates await(long time,
- TimeUnit timeUnit) throws InterruptedException,
- ExecutionException, TimeoutException {
- return result.get(time, timeUnit);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
deleted file mode 100644
index fc9af5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/RunningDatanodeState.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine.SCMConnectionManager;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.container.common.states.DatanodeState;
-import org.apache.hadoop.ozone.container.common.states.endpoint.HeartbeatEndpointTask;
-import org.apache.hadoop.ozone.container.common.states.endpoint.RegisterEndpointTask;
-import org.apache.hadoop.ozone.container.common.states.endpoint.VersionEndpointTask;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-/**
- * Class that implements handshake with SCM.
- */
-public class RunningDatanodeState implements DatanodeState {
- static final Logger
- LOG = LoggerFactory.getLogger(RunningDatanodeState.class);
- private final SCMConnectionManager connectionManager;
- private final Configuration conf;
- private final StateContext context;
- private CompletionService<EndpointStateMachine.EndPointStates> ecs;
-
- public RunningDatanodeState(Configuration conf,
- SCMConnectionManager connectionManager,
- StateContext context) {
- this.connectionManager = connectionManager;
- this.conf = conf;
- this.context = context;
- }
-
- /**
- * Reads a datanode ID from the persisted information.
- *
- * @param idPath - Path to the ID File.
- * @return DatanodeID
- * @throws IOException
- */
- private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
- readPersistedDatanodeID(Path idPath) throws IOException {
- Preconditions.checkNotNull(idPath);
- DatanodeID datanodeID = null;
- List<DatanodeID> datanodeIDs =
- ContainerUtils.readDatanodeIDsFrom(idPath.toFile());
- int containerPort = this.context.getContainerPort();
- for(DatanodeID dnId : datanodeIDs) {
- if(dnId.getContainerPort() == containerPort) {
- datanodeID = dnId;
- break;
- }
- }
-
- if (datanodeID == null) {
- throw new IOException("No valid datanode ID found from "
- + idPath.toFile().getAbsolutePath()
- + " that matches container port "
- + containerPort);
- } else {
- StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
- containerIDProto =
- StorageContainerDatanodeProtocolProtos
- .ContainerNodeIDProto
- .newBuilder()
- .setDatanodeID(datanodeID.getProtoBufMessage())
- .build();
- return containerIDProto;
- }
- }
-
- /**
- * Returns ContainerNodeIDProto or null in case of Error.
- *
- * @return ContainerNodeIDProto
- */
- private StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto
- getContainerNodeID() {
- String dataNodeIDPath = OzoneUtils.getDatanodeIDPath(conf);
- if (dataNodeIDPath == null || dataNodeIDPath.isEmpty()) {
- LOG.error("A valid file path is needed for config setting {}",
- ScmConfigKeys.OZONE_SCM_DATANODE_ID);
-
- // This is an unrecoverable error.
- this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
- return null;
- }
- StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto nodeID;
- // try to read an existing ContainerNode ID.
- try {
- nodeID = readPersistedDatanodeID(Paths.get(dataNodeIDPath));
- if (nodeID != null) {
- LOG.trace("Read Node ID :", nodeID.getDatanodeID().getDatanodeUuid());
- return nodeID;
- }
- } catch (IOException ex) {
- LOG.trace("Not able to find container Node ID, creating it.", ex);
- }
- this.context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
- return null;
- }
-
- /**
- * Called before entering this state.
- */
- @Override
- public void onEnter() {
- LOG.trace("Entering handshake task.");
- }
-
- /**
- * Called After exiting this state.
- */
- @Override
- public void onExit() {
- LOG.trace("Exiting handshake task.");
- }
-
- /**
- * Executes one or more tasks that is needed by this state.
- *
- * @param executor - ExecutorService
- */
- @Override
- public void execute(ExecutorService executor) {
- ecs = new ExecutorCompletionService<>(executor);
- for (EndpointStateMachine endpoint : connectionManager.getValues()) {
- Callable<EndpointStateMachine.EndPointStates> endpointTask
- = getEndPointTask(endpoint);
- ecs.submit(endpointTask);
- }
- }
- //TODO : Cache some of these tasks instead of creating them
- //all the time.
- private Callable<EndpointStateMachine.EndPointStates>
- getEndPointTask(EndpointStateMachine endpoint) {
- switch (endpoint.getState()) {
- case GETVERSION:
- return new VersionEndpointTask(endpoint, conf);
- case REGISTER:
- return RegisterEndpointTask.newBuilder()
- .setConfig(conf)
- .setEndpointStateMachine(endpoint)
- .setNodeID(getContainerNodeID())
- .build();
- case HEARTBEAT:
- return HeartbeatEndpointTask.newBuilder()
- .setConfig(conf)
- .setEndpointStateMachine(endpoint)
- .setNodeID(getContainerNodeID())
- .setContext(context)
- .build();
- case SHUTDOWN:
- break;
- default:
- throw new IllegalArgumentException("Illegal Argument.");
- }
- return null;
- }
-
- /**
- * Computes the next state the container state machine must move to by looking
- * at all the state of endpoints.
- * <p>
- * if any endpoint state has moved to Shutdown, either we have an
- * unrecoverable error or we have been told to shutdown. Either case the
- * datanode state machine should move to Shutdown state, otherwise we
- * remain in the Running state.
- *
- * @return next container state.
- */
- private DatanodeStateMachine.DatanodeStates
- computeNextContainerState(
- List<Future<EndpointStateMachine.EndPointStates>> results) {
- for (Future<EndpointStateMachine.EndPointStates> state : results) {
- try {
- if (state.get() == EndpointStateMachine.EndPointStates.SHUTDOWN) {
- // if any endpoint tells us to shutdown we move to shutdown state.
- return DatanodeStateMachine.DatanodeStates.SHUTDOWN;
- }
- } catch (InterruptedException | ExecutionException e) {
- LOG.error("Error in executing end point task.", e);
- }
- }
- return DatanodeStateMachine.DatanodeStates.RUNNING;
- }
-
- /**
- * Wait for execute to finish.
- *
- * @param duration - Time
- * @param timeUnit - Unit of duration.
- */
- @Override
- public DatanodeStateMachine.DatanodeStates
- await(long duration, TimeUnit timeUnit)
- throws InterruptedException, ExecutionException, TimeoutException {
- int count = connectionManager.getValues().size();
- int returned = 0;
- long timeLeft = timeUnit.toMillis(duration);
- long startTime = Time.monotonicNow();
- List<Future<EndpointStateMachine.EndPointStates>> results = new
- LinkedList<>();
-
- while (returned < count && timeLeft > 0) {
- Future<EndpointStateMachine.EndPointStates> result =
- ecs.poll(timeLeft, TimeUnit.MILLISECONDS);
- if (result != null) {
- results.add(result);
- returned++;
- }
- timeLeft = timeLeft - (Time.monotonicNow() - startTime);
- }
- return computeNextContainerState(results);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
deleted file mode 100644
index 6b8d16c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/datanode/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.datanode;
-/**
- This package contians files that guide the state transitions from
- Init->Running->Shutdown for the datanode.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
deleted file mode 100644
index b4cce23..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/HeartbeatEndpointTask.java
+++ /dev/null
@@ -1,270 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers
- .DeletedContainerBlocksSummary;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine.EndPointStates;
-import org.apache.hadoop.ozone.container.common.statemachine.StateContext;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto;
-import org.apache.hadoop.ozone.protocol.commands.SendContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.time.ZonedDateTime;
-import java.util.concurrent.Callable;
-
-/**
- * Heartbeat class for SCMs.
- */
-public class HeartbeatEndpointTask
- implements Callable<EndpointStateMachine.EndPointStates> {
- static final Logger LOG =
- LoggerFactory.getLogger(HeartbeatEndpointTask.class);
- private final EndpointStateMachine rpcEndpoint;
- private final Configuration conf;
- private ContainerNodeIDProto containerNodeIDProto;
- private StateContext context;
-
- /**
- * Constructs a SCM heart beat.
- *
- * @param conf Config.
- */
- public HeartbeatEndpointTask(EndpointStateMachine rpcEndpoint,
- Configuration conf, StateContext context) {
- this.rpcEndpoint = rpcEndpoint;
- this.conf = conf;
- this.context = context;
- }
-
- /**
- * Get the container Node ID proto.
- *
- * @return ContainerNodeIDProto
- */
- public ContainerNodeIDProto getContainerNodeIDProto() {
- return containerNodeIDProto;
- }
-
- /**
- * Set container node ID proto.
- *
- * @param containerNodeIDProto - the node id.
- */
- public void setContainerNodeIDProto(ContainerNodeIDProto
- containerNodeIDProto) {
- this.containerNodeIDProto = containerNodeIDProto;
- }
-
- /**
- * Computes a result, or throws an exception if unable to do so.
- *
- * @return computed result
- * @throws Exception if unable to compute a result
- */
- @Override
- public EndpointStateMachine.EndPointStates call() throws Exception {
- rpcEndpoint.lock();
- try {
- Preconditions.checkState(this.containerNodeIDProto != null);
- DatanodeID datanodeID = DatanodeID.getFromProtoBuf(this
- .containerNodeIDProto.getDatanodeID());
-
- SCMHeartbeatResponseProto reponse = rpcEndpoint.getEndPoint()
- .sendHeartbeat(datanodeID, this.context.getNodeReport(),
- this.context.getContainerReportState());
- processResponse(reponse, datanodeID);
- rpcEndpoint.setLastSuccessfulHeartbeat(ZonedDateTime.now());
- rpcEndpoint.zeroMissedCount();
- } catch (IOException ex) {
- rpcEndpoint.logIfNeeded(ex);
- } finally {
- rpcEndpoint.unlock();
- }
- return rpcEndpoint.getState();
- }
-
- /**
- * Returns a builder class for HeartbeatEndpointTask task.
- * @return Builder.
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Add this command to command processing Queue.
- *
- * @param response - SCMHeartbeat response.
- */
- private void processResponse(SCMHeartbeatResponseProto response,
- final DatanodeID datanodeID) {
- for (SCMCommandResponseProto commandResponseProto : response
- .getCommandsList()) {
- // Verify the response is indeed for this datanode.
- Preconditions.checkState(commandResponseProto.getDatanodeUUID()
- .equalsIgnoreCase(datanodeID.getDatanodeUuid().toString()),
- "Unexpected datanode ID in the response.");
- switch (commandResponseProto.getCmdType()) {
- case sendContainerReport:
- this.context.addCommand(SendContainerCommand.getFromProtobuf(
- commandResponseProto.getSendReport()));
- break;
- case reregisterCommand:
- if (rpcEndpoint.getState() == EndPointStates.HEARTBEAT) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received SCM notification to register."
- + " Interrupt HEARTBEAT and transit to REGISTER state.");
- }
- rpcEndpoint.setState(EndPointStates.REGISTER);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Illegal state {} found, expecting {}.",
- rpcEndpoint.getState().name(), EndPointStates.HEARTBEAT);
- }
- }
- break;
- case deleteBlocksCommand:
- DeleteBlocksCommand db = DeleteBlocksCommand
- .getFromProtobuf(commandResponseProto.getDeleteBlocksProto());
- if (!db.blocksTobeDeleted().isEmpty()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(DeletedContainerBlocksSummary
- .getFrom(db.blocksTobeDeleted())
- .toString());
- }
- this.context.addCommand(db);
- }
- break;
- case closeContainerCommand:
- CloseContainerCommand closeContainer =
- CloseContainerCommand.getFromProtobuf(
- commandResponseProto.getCloseContainerProto());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Received SCM container close request for container {}",
- closeContainer.getContainerName());
- }
- this.context.addCommand(closeContainer);
- break;
- default:
- throw new IllegalArgumentException("Unknown response : "
- + commandResponseProto.getCmdType().name());
- }
- }
- }
-
- /**
- * Builder class for HeartbeatEndpointTask.
- */
- public static class Builder {
- private EndpointStateMachine endPointStateMachine;
- private Configuration conf;
- private ContainerNodeIDProto containerNodeIDProto;
- private StateContext context;
-
- /**
- * Constructs the builder class.
- */
- public Builder() {
- }
-
- /**
- * Sets the endpoint state machine.
- *
- * @param rpcEndPoint - Endpoint state machine.
- * @return Builder
- */
- public Builder setEndpointStateMachine(EndpointStateMachine rpcEndPoint) {
- this.endPointStateMachine = rpcEndPoint;
- return this;
- }
-
- /**
- * Sets the Config.
- *
- * @param config - config
- * @return Builder
- */
- public Builder setConfig(Configuration config) {
- this.conf = config;
- return this;
- }
-
- /**
- * Sets the NodeID.
- *
- * @param nodeID - NodeID proto
- * @return Builder
- */
- public Builder setNodeID(ContainerNodeIDProto nodeID) {
- this.containerNodeIDProto = nodeID;
- return this;
- }
-
- /**
- * Sets the context.
- * @param stateContext - State context.
- * @return this.
- */
- public Builder setContext(StateContext stateContext) {
- this.context = stateContext;
- return this;
- }
-
- public HeartbeatEndpointTask build() {
- if (endPointStateMachine == null) {
- LOG.error("No endpoint specified.");
- throw new IllegalArgumentException("A valid endpoint state machine is" +
- " needed to construct HeartbeatEndpointTask task");
- }
-
- if (conf == null) {
- LOG.error("No config specified.");
- throw new IllegalArgumentException("A valid configration is needed to" +
- " construct HeartbeatEndpointTask task");
- }
-
- if (containerNodeIDProto == null) {
- LOG.error("No nodeID specified.");
- throw new IllegalArgumentException("A vaild Node ID is needed to " +
- "construct HeartbeatEndpointTask task");
- }
-
- HeartbeatEndpointTask task = new HeartbeatEndpointTask(this
- .endPointStateMachine, this.conf, this.context);
- task.setContainerNodeIDProto(containerNodeIDProto);
- return task;
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
deleted file mode 100644
index 30f87e24..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
+++ /dev/null
@@ -1,456 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import static java.util.concurrent.TimeUnit.SECONDS;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.*;
-import java.util.concurrent.TimeoutException;
-
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-
-/**
- * Tests for Local Cache Buffer Manager.
- */
-public class TestBufferManager {
- private final static long GB = 1024 * 1024 * 1024;
- private final static int KB = 1024;
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration config;
- private static StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private static XceiverClientManager xceiverClientManager;
-
- @BeforeClass
- public static void init() throws IOException {
- config = new OzoneConfiguration();
- String path = GenericTestUtils.getTempPath(
- TestBufferManager.class.getSimpleName());
- config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- cluster = new MiniOzoneClassicCluster.Builder(config)
- .numDataNodes(1).setHandlerType("distributed").build();
- storageContainerLocationClient = cluster
- .createStorageContainerLocationClient();
- xceiverClientManager = new XceiverClientManager(config);
- }
-
- @AfterClass
- public static void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
- }
-
- /**
- * createContainerAndGetPipeline creates a set of containers and returns the
- * Pipelines that define those containers.
- *
- * @param count - Number of containers to create.
- * @return - List of Pipelines.
- * @throws IOException
- */
- private List<Pipeline> createContainerAndGetPipeline(int count)
- throws IOException {
- List<Pipeline> containerPipelines = new LinkedList<>();
- for (int x = 0; x < count; x++) {
- String traceID = "trace" + RandomStringUtils.randomNumeric(4);
- String containerName = "container" + RandomStringUtils.randomNumeric(10);
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerName, "CBLOCK");
- XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
- ContainerProtocolCalls.createContainer(client, traceID);
- // This step is needed since we set private data on pipelines, when we
- // read the list from CBlockServer. So we mimic that action here.
- pipeline.setData(Longs.toByteArray(x));
- containerPipelines.add(pipeline);
- xceiverClientManager.releaseClient(client);
- }
- return containerPipelines;
- }
-
- /**
- * This test writes some block to the cache and then shuts down the cache.
- * The cache is then restarted to check that the
- * correct number of blocks are read from Dirty Log
- *
- * @throws IOException
- */
- @Test
- public void testEmptyBlockBufferHandling() throws IOException,
- InterruptedException, TimeoutException {
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- List<Pipeline> pipelines = createContainerAndGetPipeline(10);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- // Write data to the cache
- cache.put(1, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
- Assert.assertEquals(1, metrics.getNumWriteOps());
- cache.put(2, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
- Assert.assertEquals(2, metrics.getNumWriteOps());
-
- // Store the previous block buffer position
- Assert.assertEquals(2, metrics.getNumBlockBufferUpdates());
- // Simulate a shutdown by closing the cache
- cache.close();
- Thread.sleep(1000);
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
- Assert.assertEquals(2 * (Long.SIZE/ Byte.SIZE),
- metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
- Assert.assertEquals(0, metrics.getNumInterruptedBufferWaits());
-
- // Restart cache and check that right number of entries are read
- CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher newFlusher =
- new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, newMetrics);
- CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(newFlusher)
- .setCBlockTargetMetrics(newMetrics)
- .build();
- newCache.start();
- Thread fllushListenerThread = new Thread(newFlusher);
- fllushListenerThread.setDaemon(true);
- fllushListenerThread.start();
-
- Thread.sleep(5000);
- Assert.assertEquals(metrics.getNumBlockBufferUpdates(),
- newMetrics.getNumDirtyLogBlockRead());
- Assert.assertEquals(newMetrics.getNumDirtyLogBlockRead()
- * (Long.SIZE/ Byte.SIZE), newMetrics.getNumBytesDirtyLogReads());
- // Now shutdown again, nothing should be flushed
- newFlusher.shutdown();
- Assert.assertEquals(0, newMetrics.getNumBlockBufferUpdates());
- Assert.assertEquals(0, newMetrics.getNumBytesDirtyLogWritten());
- }
-
- @Test
- public void testPeriodicFlush() throws IOException,
- InterruptedException, TimeoutException{
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- flushTestConfig
- .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 5, SECONDS);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(createContainerAndGetPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- Thread.sleep(8000);
- // Ticks will be at 5s, 10s and so on, so this count should be 1
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
- // Nothing pushed to cache, so nothing should be written
- Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
- cache.close();
- // After close, another trigger should happen but still no data written
- Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
- Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
- Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
- }
-
- @Test
- public void testSingleBufferFlush() throws IOException,
- InterruptedException, TimeoutException {
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(createContainerAndGetPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
-
- for (int i = 0; i < 511; i++) {
- cache.put(i, data.getBytes(StandardCharsets.UTF_8));
- }
- // After writing 511 block no flush should happen
- Assert.assertEquals(0, metrics.getNumBlockBufferFlushTriggered());
- Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
-
-
- // After one more block it should
- cache.put(512, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
- Thread.sleep(1000);
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
- cache.close();
- Assert.assertEquals(512 * (Long.SIZE / Byte.SIZE),
- metrics.getNumBytesDirtyLogWritten());
- }
-
- @Test
- public void testMultipleBuffersFlush() throws IOException,
- InterruptedException, TimeoutException {
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- flushTestConfig
- .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 120, SECONDS);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(createContainerAndGetPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
-
- for (int i = 0; i < 4; i++) {
- for (int j = 0; j < 512; j++) {
- cache.put(i * 512 + j, data.getBytes(StandardCharsets.UTF_8));
- }
- // Flush should be triggered after every 512 block write
- Assert.assertEquals(i + 1, metrics.getNumBlockBufferFlushTriggered());
- }
- Assert.assertEquals(0, metrics.getNumIllegalDirtyLogFiles());
- Assert.assertEquals(0, metrics.getNumFailedDirtyLogFileDeletes());
- cache.close();
- Assert.assertEquals(4 * 512 * (Long.SIZE / Byte.SIZE),
- metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(5, metrics.getNumBlockBufferFlushTriggered());
- Assert.assertEquals(4, metrics.getNumBlockBufferFlushCompleted());
- }
-
- @Test
- public void testSingleBlockFlush() throws IOException,
- InterruptedException, TimeoutException{
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- flushTestConfig
- .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
- 5, SECONDS);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(createContainerAndGetPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- cache.put(0, data.getBytes(StandardCharsets.UTF_8));
- Thread.sleep(8000);
- // Ticks will be at 5s, 10s and so on, so this count should be 1
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
- // 1 block written to cache, which should be flushed
- Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
- cache.close();
- // After close, another trigger should happen but no data should be written
- Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
- Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
- Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
- }
-
- @Test
- public void testRepeatedBlockWrites() throws IOException,
- InterruptedException, TimeoutException{
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestBufferManager.class.getSimpleName()
- + RandomStringUtils.randomNumeric(4));
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(createContainerAndGetPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- Thread fllushListenerThread = new Thread(flusher);
- fllushListenerThread.setDaemon(true);
- fllushListenerThread.start();
- cache.start();
- for (int i = 0; i < 512; i++) {
- cache.put(i, data.getBytes(StandardCharsets.UTF_8));
- }
- Assert.assertEquals(512, metrics.getNumWriteOps());
- Assert.assertEquals(512, metrics.getNumBlockBufferUpdates());
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
- Thread.sleep(5000);
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
-
-
- for (int i = 0; i < 512; i++) {
- cache.put(i, data.getBytes(StandardCharsets.UTF_8));
- }
- Assert.assertEquals(1024, metrics.getNumWriteOps());
- Assert.assertEquals(1024, metrics.getNumBlockBufferUpdates());
- Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
-
- Thread.sleep(5000);
- Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
- Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
- Assert.assertEquals(2, metrics.getNumBlockBufferFlushCompleted());
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
deleted file mode 100644
index f8b05ed..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockCLI.java
+++ /dev/null
@@ -1,242 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.hadoop.cblock.cli.CBlockCli;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.ByteArrayOutputStream;
-import java.io.File;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * A testing class for cblock command line tool.
- */
-public class TestCBlockCLI {
- private static final long GB = 1 * 1024 * 1024 * 1024L;
- private static final int KB = 1024;
- private static CBlockCli cmd;
- private static OzoneConfiguration conf;
- private static CBlockManager cBlockManager;
- private static ByteArrayOutputStream outContent;
- private static PrintStream testPrintOut;
-
- @BeforeClass
- public static void setup() throws IOException {
- outContent = new ByteArrayOutputStream();
- ScmClient storageClient = new MockStorageClient();
- conf = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestCBlockCLI.class.getSimpleName());
- File filePath = new File(path);
- if (!filePath.exists() && !filePath.mkdirs()) {
- throw new IOException("Unable to create test DB dir");
- }
- conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
- "/testCblockCli.dat"));
- cBlockManager = new CBlockManager(conf, storageClient);
- cBlockManager.start();
- testPrintOut = new PrintStream(outContent);
- cmd = new CBlockCli(conf, testPrintOut);
- }
-
- @AfterClass
- public static void clean() {
- if (cBlockManager != null) {
- cBlockManager.stop();
- cBlockManager.join();
- cBlockManager.clean();
- }
- }
-
- @After
- public void reset() {
- outContent.reset();
- }
-
- /**
- * Test the help command.
- * @throws Exception
- */
- @Test
- public void testCliHelp() throws Exception {
- PrintStream initialStdOut = System.out;
- System.setOut(testPrintOut);
- String[] args = {"-h"};
- cmd.run(args);
- String helpPrints =
- "usage: cblock\n" +
- " -c,--createVolume <user> <volume> <volumeSize in [GB/TB]> " +
- "<blockSize> create a fresh new volume\n" +
- " -d,--deleteVolume <user> <volume> " +
- " delete a volume\n" +
- " -h,--help " +
- " help\n" +
- " -i,--infoVolume <user> <volume> " +
- " info a volume\n" +
- " -l,--listVolume <user> " +
- " list all volumes\n" +
- " -s,--serverAddr <serverAddress>:<serverPort> " +
- " specify server address:port\n";
- assertEquals(helpPrints, outContent.toString());
- outContent.reset();
- System.setOut(initialStdOut);
- }
-
- /**
- * Test volume listing command.
- * @throws Exception
- */
- @Test
- public void testCliList() throws Exception {
- String userName0 = "userTestCliList0";
- String userName1 = "userTestCliList1";
- String userTestNotExist = "userTestNotExist";
- String volumeName0 = "volumeTest0";
- String volumeName1 = "volumeTest1";
- String volumeSize0 = "30GB";
- String volumeSize1 = "40GB";
- String blockSize = Integer.toString(4);
- String[] argsCreate0 =
- {"-c", userName0, volumeName0, volumeSize0, blockSize};
- cmd.run(argsCreate0);
- String[] argsCreate1 =
- {"-c", userName0, volumeName1, volumeSize1, blockSize};
- cmd.run(argsCreate1);
- String[] argsCreate2 =
- {"-c", userName1, volumeName0, volumeSize0, blockSize};
- cmd.run(argsCreate2);
- String[] argsList0 = {"-l"};
- cmd.run(argsList0);
- String[] outExpected1 = {
- "userTestCliList1:volumeTest0\t32212254720\t4096\n",
- "userTestCliList0:volumeTest0\t32212254720\t4096\n",
- "userTestCliList0:volumeTest1\t42949672960\t4096\n"};
- int length = 0;
- for (String str : outExpected1) {
- assertTrue(outContent.toString().contains(str));
- length += str.length();
- }
- assertEquals(length, outContent.toString().length());
- outContent.reset();
-
- String[] argsList1 = {"-l", userName1};
- cmd.run(argsList1);
- String outExpected2 = "userTestCliList1:volumeTest0\t32212254720\t4096\n";
- assertEquals(outExpected2, outContent.toString());
- outContent.reset();
-
- String[] argsList2 = {"-l", userTestNotExist};
- cmd.run(argsList2);
- String outExpected3 = "\n";
- assertEquals(outExpected3, outContent.toString());
- }
-
- /**
- * Test create volume command.
- * @throws Exception
- */
- @Test
- public void testCliCreate() throws Exception {
- String userName = "userTestCliCreate";
- String volumeName = "volumeTest";
- String volumeSize = "30GB";
- String blockSize = "4";
- String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
- cmd.run(argsCreate);
- List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
- assertEquals(1, allVolumes.size());
- VolumeDescriptor volume = allVolumes.get(0);
- assertEquals(userName, volume.getUserName());
- assertEquals(volumeName, volume.getVolumeName());
- long volumeSizeB = volume.getVolumeSize();
- assertEquals(30, (int)(volumeSizeB/ GB));
- assertEquals(4, volume.getBlockSize()/ KB);
- }
-
- /**
- * Test delete volume command.
- * @throws Exception
- */
- @Test
- public void testCliDelete() throws Exception {
- String userName = "userTestCliDelete";
- String volumeName = "volumeTest";
- String volumeSize = "30GB";
- String blockSize = "4";
- String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
- cmd.run(argsCreate);
- List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
- assertEquals(1, allVolumes.size());
- VolumeDescriptor volume = allVolumes.get(0);
- assertEquals(userName, volume.getUserName());
- assertEquals(volumeName, volume.getVolumeName());
- long volumeSizeB = volume.getVolumeSize();
- assertEquals(30, (int)(volumeSizeB/ GB));
- assertEquals(4, volume.getBlockSize()/ KB);
-
- String[] argsDelete = {"-d", userName, volumeName};
- cmd.run(argsDelete);
- allVolumes = cBlockManager.getAllVolumes(userName);
- assertEquals(0, allVolumes.size());
- }
-
- /**
- * Test info volume command.
- * @throws Exception
- */
- @Test
- public void testCliInfoVolume() throws Exception {
- String userName0 = "userTestCliInfo";
- String volumeName0 = "volumeTest0";
- String volumeSize = "8000GB";
- String blockSize = "4";
- String[] argsCreate0 = {
- "-c", userName0, volumeName0, volumeSize, blockSize};
- cmd.run(argsCreate0);
- String[] argsInfo = {"-i", userName0, volumeName0};
- cmd.run(argsInfo);
- // TODO : the usage field is not implemented yet, always 0 now.
- String outExpected = " userName:userTestCliInfo " +
- "volumeName:volumeTest0 " +
- "volumeSize:8589934592000 " +
- "blockSize:4096 (sizeInBlocks:2097152000) usageInBlocks:0\n";
- assertEquals(outExpected, outContent.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
deleted file mode 100644
index cdd65c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
+++ /dev/null
@@ -1,377 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-
-/**
- * Tests for Cblock read write functionality.
- */
-public class TestCBlockReadWrite {
- private final static long GB = 1024 * 1024 * 1024;
- private final static int KB = 1024;
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration config;
- private static StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private static XceiverClientManager xceiverClientManager;
-
- @BeforeClass
- public static void init() throws IOException {
- config = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestCBlockReadWrite.class.getSimpleName());
- config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- cluster = new MiniOzoneClassicCluster.Builder(config)
- .numDataNodes(1)
- .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
- storageContainerLocationClient = cluster
- .createStorageContainerLocationClient();
- xceiverClientManager = new XceiverClientManager(config);
- }
-
- @AfterClass
- public static void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
- }
-
- /**
- * getContainerPipelines creates a set of containers and returns the
- * Pipelines that define those containers.
- *
- * @param count - Number of containers to create.
- * @return - List of Pipelines.
- * @throws IOException throws Exception
- */
- private List<Pipeline> getContainerPipeline(int count) throws IOException {
- List<Pipeline> containerPipelines = new LinkedList<>();
- for (int x = 0; x < count; x++) {
- String traceID = "trace" + RandomStringUtils.randomNumeric(4);
- String containerName = "container" + RandomStringUtils.randomNumeric(10);
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerName, "CBLOCK");
- XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
- ContainerProtocolCalls.createContainer(client, traceID);
- // This step is needed since we set private data on pipelines, when we
- // read the list from CBlockServer. So we mimic that action here.
- pipeline.setData(Longs.toByteArray(x));
- containerPipelines.add(pipeline);
- }
- return containerPipelines;
- }
-
- /**
- * This test creates a cache and performs a simple write / read.
- * The operations are done by bypassing the cache.
- *
- * @throws IOException
- */
- @Test
- public void testDirectIO() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration cConfig = new OzoneConfiguration();
- cConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
- cConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- final long blockID = 0;
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- String dataHash = DigestUtils.sha256Hex(data);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(cConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(cConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- Assert.assertFalse(cache.isShortCircuitIOEnabled());
- cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(1, metrics.getNumDirectBlockWrites());
- Assert.assertEquals(1, metrics.getNumWriteOps());
- // Please note that this read is directly from remote container
- LogicalBlock block = cache.get(blockID);
- Assert.assertEquals(1, metrics.getNumReadOps());
- Assert.assertEquals(0, metrics.getNumReadCacheHits());
- Assert.assertEquals(1, metrics.getNumReadCacheMiss());
- Assert.assertEquals(0, metrics.getNumReadLostBlocks());
- Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
-
- cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(2, metrics.getNumDirectBlockWrites());
- Assert.assertEquals(2, metrics.getNumWriteOps());
- Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
- // Please note that this read is directly from remote container
- block = cache.get(blockID + 1);
- Assert.assertEquals(2, metrics.getNumReadOps());
- Assert.assertEquals(0, metrics.getNumReadCacheHits());
- Assert.assertEquals(2, metrics.getNumReadCacheMiss());
- Assert.assertEquals(0, metrics.getNumReadLostBlocks());
- String readHash = DigestUtils.sha256Hex(block.getData().array());
- Assert.assertEquals("File content does not match.", dataHash, readHash);
- GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
- cache.close();
- }
-
- /**
- * This test writes some block to the cache and then shuts down the cache
- * The cache is then restarted with "short.circuit.io" disable to check
- * that the blocks are read correctly from the container.
- *
- * @throws IOException
- */
- @Test
- public void testContainerWrites() throws IOException,
- InterruptedException, TimeoutException {
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestCBlockReadWrite.class.getSimpleName());
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 3,
- TimeUnit.SECONDS);
- XceiverClientManager xcm = new XceiverClientManager(flushTestConfig);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
-
- int numUniqueBlocks = 4;
- String[] data = new String[numUniqueBlocks];
- String[] dataHash = new String[numUniqueBlocks];
- for (int i = 0; i < numUniqueBlocks; i++) {
- data[i] = RandomStringUtils.random(4 * KB);
- dataHash[i] = DigestUtils.sha256Hex(data[i]);
- }
-
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xcm, metrics);
- List<Pipeline> pipelines = getContainerPipeline(10);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xcm)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- Thread flushListenerThread = new Thread(flusher);
- flushListenerThread.setDaemon(true);
- flushListenerThread.start();
- Assert.assertTrue(cache.isShortCircuitIOEnabled());
- // Write data to the cache
- for (int i = 0; i < 512; i++) {
- cache.put(i, data[i % numUniqueBlocks].getBytes(StandardCharsets.UTF_8));
- }
- // Close the cache and flush the data to the containers
- cache.close();
- Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
- Assert.assertEquals(512, metrics.getNumWriteOps());
- Thread.sleep(3000);
- flusher.shutdown();
- Assert.assertTrue(metrics.getNumBlockBufferFlushTriggered() > 1);
- Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
- Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
- Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
- Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
- // Now disable DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO and restart cache
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
- CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher newFlusher =
- new ContainerCacheFlusher(flushTestConfig, xcm, newMetrics);
- CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xcm)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(newFlusher)
- .setCBlockTargetMetrics(newMetrics)
- .build();
- newCache.start();
- Assert.assertFalse(newCache.isShortCircuitIOEnabled());
- // this read will be from the container, also match the hash
- for (int i = 0; i < 512; i++) {
- LogicalBlock block = newCache.get(i);
- String readHash = DigestUtils.sha256Hex(block.getData().array());
- Assert.assertEquals("File content does not match, for index:"
- + i, dataHash[i % numUniqueBlocks], readHash);
- }
- Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
- Assert.assertEquals(0, newMetrics.getNumFailedReadBlocks());
- newCache.close();
- newFlusher.shutdown();
- }
-
- @Test
- public void testRetryLog() throws IOException,
- InterruptedException, TimeoutException {
- // Create a new config so that this tests write metafile to new location
- OzoneConfiguration flushTestConfig = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestCBlockReadWrite.class.getSimpleName());
- flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
- 3,
- TimeUnit.SECONDS);
-
- int numblocks = 10;
- flushTestConfig.setInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, numblocks);
-
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
-
- List<Pipeline> fakeContainerPipelines = new LinkedList<>();
- PipelineChannel pipelineChannel = new PipelineChannel("fake",
- LifeCycleState.OPEN, ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
- "fake");
- Pipeline fakePipeline = new Pipeline("fake", pipelineChannel);
- fakePipeline.setData(Longs.toByteArray(1));
- fakeContainerPipelines.add(fakePipeline);
-
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(fakeContainerPipelines)
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- Thread flushListenerThread = new Thread(flusher);
- flushListenerThread.setDaemon(true);
- flushListenerThread.start();
-
- for (int i = 0; i < numblocks; i++) {
- cache.put(i, data.getBytes(StandardCharsets.UTF_8));
- }
- Assert.assertEquals(numblocks, metrics.getNumWriteOps());
- Thread.sleep(3000);
-
- // all the writes to the container will fail because of fake pipelines
- Assert.assertEquals(numblocks, metrics.getNumDirtyLogBlockRead());
- Assert.assertTrue(
- metrics.getNumWriteGenericExceptionRetryBlocks() >= numblocks);
- Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
- Assert.assertEquals(0, metrics.getNumFailedRetryLogFileWrites());
- Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
- cache.close();
- flusher.shutdown();
-
- // restart cache with correct pipelines, now blocks should be uploaded
- // correctly
- CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher newFlusher =
- new ContainerCacheFlusher(flushTestConfig,
- xceiverClientManager, newMetrics);
- CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
- .setConfiguration(flushTestConfig)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(newFlusher)
- .setCBlockTargetMetrics(newMetrics)
- .build();
- newCache.start();
- Thread newFlushListenerThread = new Thread(newFlusher);
- newFlushListenerThread.setDaemon(true);
- newFlushListenerThread.start();
- Thread.sleep(3000);
- Assert.assertTrue(newMetrics.getNumRetryLogBlockRead() >= numblocks);
- Assert.assertEquals(0, newMetrics.getNumWriteGenericExceptionRetryBlocks());
- Assert.assertEquals(0, newMetrics.getNumWriteIOExceptionRetryBlocks());
- Assert.assertEquals(0, newMetrics.getNumFailedReleaseLevelDB());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
deleted file mode 100644
index b5695fc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * This class tests the basics of CBlock server. Mainly about the four
- * operations on volumes: create, delete, info and list.
- */
-public class TestCBlockServer {
- private static CBlockManager cBlockManager;
- private static OzoneConfiguration conf;
-
- @Before
- public void setup() throws Exception {
- ScmClient storageClient = new MockStorageClient();
- conf = new OzoneConfiguration();
- conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
- cBlockManager = new CBlockManager(conf, storageClient);
- cBlockManager.start();
- }
-
- @After
- public void clean() {
- cBlockManager.stop();
- cBlockManager.join();
- cBlockManager.clean();
- }
-
- /**
- * Test create volume for different users.
- * @throws Exception
- */
- @Test
- public void testCreateVolume() throws Exception {
- String userName1 = "user" + RandomStringUtils.randomNumeric(5);
- String userName2 = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5);
- String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 1L*1024*1024;
- int blockSize = 4096;
- cBlockManager.createVolume(userName1, volumeName1, volumeSize, blockSize);
- List<VolumeInfo> volumes = cBlockManager.listVolume(userName1);
- assertEquals(1, volumes.size());
- VolumeInfo existingVolume = volumes.get(0);
- assertEquals(userName1, existingVolume.getUserName());
- assertEquals(volumeName1, existingVolume.getVolumeName());
- assertEquals(volumeSize, existingVolume.getVolumeSize());
- assertEquals(blockSize, existingVolume.getBlockSize());
-
- cBlockManager.createVolume(userName1, volumeName2, volumeSize, blockSize);
- cBlockManager.createVolume(userName2, volumeName1, volumeSize, blockSize);
- volumes = cBlockManager.listVolume(userName1);
- assertEquals(2, volumes.size());
- volumes = cBlockManager.listVolume(userName2);
- assertEquals(1, volumes.size());
- }
-
- /**
- * Test delete volume.
- * @throws Exception
- */
- @Test
- public void testDeleteVolume() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5);
- String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 1L*1024*1024;
- int blockSize = 4096;
- cBlockManager.createVolume(userName, volumeName1, volumeSize, blockSize);
- cBlockManager.createVolume(userName, volumeName2, volumeSize, blockSize);
- cBlockManager.deleteVolume(userName, volumeName1, true);
- List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
- assertEquals(1, volumes.size());
-
- VolumeInfo existingVolume = volumes.get(0);
- assertEquals(userName, existingVolume.getUserName());
- assertEquals(volumeName2, existingVolume.getVolumeName());
- assertEquals(volumeSize, existingVolume.getVolumeSize());
- assertEquals(blockSize, existingVolume.getBlockSize());
- }
-
- /**
- * Test info volume.
- *
- * TODO : usage field is not being tested (as it is not implemented yet)
- * @throws Exception
- */
- @Test
- public void testInfoVolume() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 1L*1024*1024;
- int blockSize = 4096;
- cBlockManager.createVolume(userName, volumeName, volumeSize, blockSize);
- VolumeInfo info = cBlockManager.infoVolume(userName, volumeName);
- assertEquals(userName, info.getUserName());
- assertEquals(volumeName, info.getVolumeName());
- assertEquals(volumeSize, info.getVolumeSize());
- assertEquals(blockSize, info.getBlockSize());
- }
-
- /**
- * Test listing a number of volumes.
- * @throws Exception
- */
- @Test
- public void testListVolume() throws Exception {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName ="volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 1L*1024*1024;
- int blockSize = 4096;
- int volumeNum = 100;
- for (int i = 0; i<volumeNum; i++) {
- cBlockManager.createVolume(userName, volumeName + i,
- volumeSize, blockSize);
- }
- List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
- assertEquals(volumeNum, volumes.size());
- Set<String> volumeIds = new HashSet<>();
- for (int i = 0; i<volumeNum; i++) {
- VolumeInfo volumeInfo = volumes.get(i);
- assertEquals(userName, volumeInfo.getUserName());
- assertFalse(volumeIds.contains(volumeName + i));
- volumeIds.add(volumeName + i);
- assertEquals(volumeSize, volumeInfo.getVolumeSize());
- assertEquals(blockSize, volumeInfo.getBlockSize());
- }
- for (int i = 0; i<volumeNum; i++) {
- assertTrue(volumeIds.contains(volumeName + i));
- }
- }
-
- /**
- * Test listing a number of volumes.
- * @throws Exception
- */
- @Test
- public void testListVolumes() throws Exception {
- String volumeName ="volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 1L*1024*1024;
- int blockSize = 4096;
- int volumeNum = 100;
- int userCount = 10;
-
- assertTrue("We need at least one volume for each user",
- userCount < volumeNum);
-
- for (int i = 0; i<volumeNum; i++) {
- String userName =
- "user-" + (i % userCount);
- cBlockManager.createVolume(userName, volumeName + i,
- volumeSize, blockSize);
- }
- List<VolumeInfo> allVolumes = cBlockManager.listVolumes();
- //check if we have the volumes from all the users.
-
- Set<String> volumeIds = new HashSet<>();
- Set<String> usernames = new HashSet<>();
- for (int i = 0; i < allVolumes.size(); i++) {
- VolumeInfo volumeInfo = allVolumes.get(i);
- assertFalse(volumeIds.contains(volumeName + i));
- usernames.add(volumeInfo.getUserName());
- volumeIds.add(volumeName + i);
- assertEquals(volumeSize, volumeInfo.getVolumeSize());
- assertEquals(blockSize, volumeInfo.getBlockSize());
- }
-
- assertEquals(volumeNum, volumeIds.size());
- for (int i = 0; i<volumeNum; i++) {
- assertTrue(volumeIds.contains(volumeName + i));
- }
-
- assertEquals(userCount, usernames.size());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
deleted file mode 100644
index c6896a1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.cblock.util.MockStorageClient;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.Test;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test the CBlock server state is maintained in persistent storage and can be
- * recovered on CBlock server restart.
- */
-public class TestCBlockServerPersistence {
-
- /**
- * Test when cblock server fails with volume meta data, the meta data can be
- * restored correctly.
- * @throws Exception
- */
- @Test
- public void testWriteToPersistentStore() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- String userName = "testWriteToPersistentStore";
- String volumeName1 = "testVolume1";
- String volumeName2 = "testVolume2";
- long volumeSize1 = 30L*1024*1024*1024;
- long volumeSize2 = 15L*1024*1024*1024;
- int blockSize = 4096;
- CBlockManager cBlockManager = null;
- CBlockManager cBlockManager1 = null;
- String path = GenericTestUtils
- .getTempPath(TestCBlockServerPersistence.class.getSimpleName());
- File filePath = new File(path);
- if(!filePath.exists() && !filePath.mkdirs()) {
- throw new IOException("Unable to create test DB dir");
- }
- conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
- "/testCblockPersistence.dat"));
- try {
- ScmClient storageClient = new MockStorageClient();
- conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
- conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
- cBlockManager = new CBlockManager(conf, storageClient);
- cBlockManager.start();
- cBlockManager.createVolume(userName, volumeName1, volumeSize1, blockSize);
- cBlockManager.createVolume(userName, volumeName2, volumeSize2, blockSize);
- List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes();
- // close the cblock server. Since meta data is written to disk on volume
- // creation, closing server here is the same as a cblock server crash.
- cBlockManager.close();
- cBlockManager.stop();
- cBlockManager.join();
- cBlockManager = null;
- assertEquals(2, allVolumes.size());
- VolumeDescriptor volumeDescriptor1 = allVolumes.get(0);
- VolumeDescriptor volumeDescriptor2 = allVolumes.get(1);
-
- // create a new cblock server instance. This is just the
- // same as restarting cblock server.
- ScmClient storageClient1 = new MockStorageClient();
- OzoneConfiguration conf1 = new OzoneConfiguration();
- conf1.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
- "/testCblockPersistence.dat"));
- conf1.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
- conf1.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
- cBlockManager1 = new CBlockManager(conf1, storageClient1);
- cBlockManager1.start();
- List<VolumeDescriptor> allVolumes1 = cBlockManager1.getAllVolumes();
- assertEquals(2, allVolumes1.size());
- VolumeDescriptor newvolumeDescriptor1 = allVolumes1.get(0);
- VolumeDescriptor newvolumeDescriptor2 = allVolumes1.get(1);
-
- // It seems levelDB iterator gets keys in the same order as keys
- // are inserted, in which case the else clause should never happen.
- // But still kept the second clause if it is possible to get different
- // key ordering from leveldb. And we do not rely on the ordering of keys
- // here.
- if (volumeDescriptor1.getVolumeName().equals(
- newvolumeDescriptor1.getVolumeName())) {
- assertEquals(volumeDescriptor1.toString(),
- newvolumeDescriptor1.toString());
- assertEquals(volumeDescriptor2.toString(),
- newvolumeDescriptor2.toString());
- } else {
- assertEquals(volumeDescriptor1.toString(),
- newvolumeDescriptor2.toString());
- assertEquals(volumeDescriptor2.toString(),
- newvolumeDescriptor1.toString());
- }
- } finally {
- if (cBlockManager != null) {
- cBlockManager.clean();
- }
- if (cBlockManager1 != null) {
- cBlockManager1.close();
- cBlockManager1.stop();
- cBlockManager1.join();
- cBlockManager1.clean();
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
deleted file mode 100644
index 7749432..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
+++ /dev/null
@@ -1,444 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockIStorageImpl;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.hadoop.util.Time;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import static java.lang.Math.abs;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_TRACE_IO;
-
-/**
- * Tests for local cache.
- */
-public class TestLocalBlockCache {
- private static final Logger LOG =
- LoggerFactory.getLogger(TestLocalBlockCache.class);
- private final static long GB = 1024 * 1024 * 1024;
- private final static int KB = 1024;
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration config;
- private static StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private static XceiverClientManager xceiverClientManager;
-
- @BeforeClass
- public static void init() throws IOException {
- config = new OzoneConfiguration();
- String path = GenericTestUtils
- .getTempPath(TestLocalBlockCache.class.getSimpleName());
- config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
- config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
- cluster = new MiniOzoneClassicCluster.Builder(config)
- .numDataNodes(1)
- .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
- storageContainerLocationClient = cluster
- .createStorageContainerLocationClient();
- xceiverClientManager = new XceiverClientManager(config);
- }
-
- @AfterClass
- public static void shutdown() throws InterruptedException {
- if (cluster != null) {
- cluster.shutdown();
- }
- IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
- }
-
- /**
- * getContainerPipelines creates a set of containers and returns the
- * Pipelines that define those containers.
- *
- * @param count - Number of containers to create.
- * @return - List of Pipelines.
- * @throws IOException throws Exception
- */
- private List<Pipeline> getContainerPipeline(int count) throws IOException {
- List<Pipeline> containerPipelines = new LinkedList<>();
- for (int x = 0; x < count; x++) {
- String traceID = "trace" + RandomStringUtils.randomNumeric(4);
- String containerName = "container" + RandomStringUtils.randomNumeric(10);
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerName, "CBLOCK");
- XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
- ContainerProtocolCalls.createContainer(client, traceID);
- // This step is needed since we set private data on pipelines, when we
- // read the list from CBlockServer. So we mimic that action here.
- pipeline.setData(Longs.toByteArray(x));
- containerPipelines.add(pipeline);
- xceiverClientManager.releaseClient(client);
- }
- return containerPipelines;
- }
-
- /**
- * This test creates a cache and performs a simple write / read.
- * Due to the cache - we have Read-after-write consistency for cBlocks.
- *
- * @throws IOException throws Exception
- */
- @Test
- public void testCacheWriteRead() throws IOException,
- InterruptedException, TimeoutException {
- final long blockID = 0;
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- String dataHash = DigestUtils.sha256Hex(data);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(1, metrics.getNumWriteOps());
- // Please note that this read is from the local cache.
- LogicalBlock block = cache.get(blockID);
- Assert.assertEquals(1, metrics.getNumReadOps());
- Assert.assertEquals(1, metrics.getNumReadCacheHits());
- Assert.assertEquals(0, metrics.getNumReadCacheMiss());
- Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-
- cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
- Assert.assertEquals(2, metrics.getNumWriteOps());
- // Please note that this read is from the local cache.
- block = cache.get(blockID + 1);
- Assert.assertEquals(2, metrics.getNumReadOps());
- Assert.assertEquals(2, metrics.getNumReadCacheHits());
- Assert.assertEquals(0, metrics.getNumReadCacheMiss());
- Assert.assertEquals(0, metrics.getNumReadLostBlocks());
- String readHash = DigestUtils.sha256Hex(block.getData().array());
- Assert.assertEquals("File content does not match.", dataHash, readHash);
- GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
- cache.close();
-
- }
-
- @Test
- public void testCacheWriteToRemoteContainer() throws IOException,
- InterruptedException, TimeoutException {
- final long blockID = 0;
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
- GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
- cache.close();
- }
-
- @Test
- public void testCacheWriteToRemote50KBlocks() throws IOException,
- InterruptedException, TimeoutException {
- final long totalBlocks = 50 * 1000;
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- String data = RandomStringUtils.random(4 * KB);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * 1024)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- long startTime = Time.monotonicNow();
- for (long blockid = 0; blockid < totalBlocks; blockid++) {
- cache.put(blockid, data.getBytes(StandardCharsets.UTF_8));
- }
- Assert.assertEquals(totalBlocks, metrics.getNumWriteOps());
- Assert.assertEquals(totalBlocks, metrics.getNumBlockBufferUpdates());
- LOG.info("Wrote 50K blocks, waiting for replication to finish.");
- GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
- long endTime = Time.monotonicNow();
- LOG.info("Time taken for writing {} blocks is {} seconds", totalBlocks,
- TimeUnit.MILLISECONDS.toSeconds(endTime - startTime));
- // TODO: Read this data back.
- cache.close();
- }
-
- @Test
- public void testCacheInvalidBlock() throws IOException {
- final int blockID = 1024;
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- // Read a non-existent block ID.
- LogicalBlock block = cache.get(blockID);
- Assert.assertNotNull(block);
- Assert.assertEquals(4 * 1024, block.getData().array().length);
- Assert.assertEquals(1, metrics.getNumReadOps());
- Assert.assertEquals(1, metrics.getNumReadLostBlocks());
- Assert.assertEquals(1, metrics.getNumReadCacheMiss());
- cache.close();
- }
-
- @Test
- public void testReadWriteCorrectness() throws IOException,
- InterruptedException, TimeoutException {
- Random r = new Random();
- final int maxBlock = 12500000;
- final int blockCount = 10 * 1000;
- Map<Long, String> blockShaMap = new HashMap<>();
- List<Pipeline> pipelines = getContainerPipeline(10);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
- String userName = "user" + RandomStringUtils.randomNumeric(4);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- final CBlockLocalCache cache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- cache.start();
- for (int x = 0; x < blockCount; x++) {
- String data = RandomStringUtils.random(4 * 1024);
- String dataHash = DigestUtils.sha256Hex(data);
- long blockId = abs(r.nextInt(maxBlock));
- blockShaMap.put(blockId, dataHash);
- cache.put(blockId, data.getBytes(StandardCharsets.UTF_8));
- }
- Assert.assertEquals(blockCount, metrics.getNumWriteOps());
- GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
- LOG.info("Finished with putting blocks ..starting reading blocks back. " +
- "unique blocks : {}", blockShaMap.size());
- // Test reading from local cache.
- for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
- LogicalBlock block = cache.get(entry.getKey());
- String blockSha = DigestUtils.sha256Hex(block.getData().array());
- Assert.assertEquals("Block data is not equal", entry.getValue(),
- blockSha);
- }
- Assert.assertEquals(blockShaMap.size(), metrics.getNumReadOps());
- Assert.assertEquals(blockShaMap.size(), metrics.getNumReadCacheHits());
- Assert.assertEquals(0, metrics.getNumReadCacheMiss());
- Assert.assertEquals(0, metrics.getNumReadLostBlocks());
-
- LOG.info("Finished with reading blocks, SUCCESS.");
- // Close and discard local cache.
- cache.close();
- LOG.info("Closing the and destroying local cache");
- CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher newflusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, newMetrics);
- Assert.assertEquals(0, newMetrics.getNumReadCacheHits());
- CBlockLocalCache newCache = null;
- try {
- newCache = CBlockLocalCache.newBuilder()
- .setConfiguration(this.config)
- .setVolumeName(volumeName)
- .setUserName(userName)
- .setPipelines(pipelines)
- .setClientManager(xceiverClientManager)
- .setBlockSize(4 * KB)
- .setVolumeSize(50 * GB)
- .setFlusher(newflusher)
- .setCBlockTargetMetrics(newMetrics)
- .build();
- newCache.start();
- for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
- LogicalBlock block = newCache.get(entry.getKey());
- String blockSha = DigestUtils.sha256Hex(block.getData().array());
- Assert.assertEquals("Block data is not equal", entry.getValue(),
- blockSha);
- }
-
- Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadOps());
- Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadCacheHits());
- Assert.assertEquals(0, newMetrics.getNumReadCacheMiss());
- Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
-
- LOG.info("Finished with reading blocks from remote cache, SUCCESS.");
- } finally {
- if (newCache != null) {
- newCache.close();
- }
- }
- }
-
- @Test
- public void testStorageImplReadWrite() throws IOException,
- InterruptedException, TimeoutException {
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 50L * (1024L * 1024L * 1024L);
- int blockSize = 4096;
- byte[] data =
- RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
- .getBytes(StandardCharsets.UTF_8);
- String hash = DigestUtils.sha256Hex(data);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
- xceiverClientManager, metrics);
- CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
- .setUserName(userName)
- .setVolumeName(volumeName)
- .setVolumeSize(volumeSize)
- .setBlockSize(blockSize)
- .setContainerList(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setConf(this.config)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- ozoneStore.write(data, 0);
-
- byte[] newData = new byte[10 * 1024 * 1024];
- ozoneStore.read(newData, 0);
- String newHash = DigestUtils.sha256Hex(newData);
- Assert.assertEquals("hashes don't match.", hash, newHash);
- GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
- 100, 20 * 1000);
- ozoneStore.close();
- }
-
- //@Test
- // Disabling this test for time being since the bug in JSCSI
- // forces us always to have a local cache.
- public void testStorageImplNoLocalCache() throws IOException,
- InterruptedException, TimeoutException {
- OzoneConfiguration oConfig = new OzoneConfiguration();
- oConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
- oConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
- String userName = "user" + RandomStringUtils.randomNumeric(5);
- String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
- long volumeSize = 50L * (1024L * 1024L * 1024L);
- int blockSize = 4096;
- byte[] data =
- RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
- .getBytes(StandardCharsets.UTF_8);
- String hash = DigestUtils.sha256Hex(data);
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- ContainerCacheFlusher flusher = new ContainerCacheFlusher(oConfig,
- xceiverClientManager, metrics);
- CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
- .setUserName(userName)
- .setVolumeName(volumeName)
- .setVolumeSize(volumeSize)
- .setBlockSize(blockSize)
- .setContainerList(getContainerPipeline(10))
- .setClientManager(xceiverClientManager)
- .setConf(oConfig)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- ozoneStore.write(data, 0);
-
- byte[] newData = new byte[10 * 1024 * 1024];
- ozoneStore.read(newData, 0);
- String newHash = DigestUtils.sha256Hex(newData);
- Assert.assertEquals("hashes don't match.", hash, newHash);
- GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
- 100, 20 * 1000);
- ozoneStore.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
deleted file mode 100644
index 02c23ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.kubernetes;
-
-import io.kubernetes.client.JSON;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.nio.file.Files;
-import java.nio.file.Paths;
-
-/**
- * Test the resource generation of Dynamic Provisioner.
- */
-public class TestDynamicProvisioner {
-
- @Test
- public void persitenceVolumeBuilder() throws Exception {
-
- OzoneConfiguration conf = new OzoneConfiguration();
- conf.setStrings(DFS_CBLOCK_ISCSI_ADVERTISED_IP, "1.2.3.4");
-
- DynamicProvisioner provisioner =
- new DynamicProvisioner(conf, null);
-
- String pvc = new String(Files.readAllBytes(
- Paths.get(getClass().getResource(
- "/dynamicprovisioner/input1-pvc.json").toURI())));
-
- String pv = new String(Files.readAllBytes(
- Paths.get(getClass().getResource(
- "/dynamicprovisioner/expected1-pv.json").toURI())));
-
- JSON json = new io.kubernetes.client.JSON();
-
- V1PersistentVolumeClaim claim =
- json.getGson().fromJson(pvc, V1PersistentVolumeClaim.class);
-
- String volumeName = provisioner.createVolumeName(claim);
-
- V1PersistentVolume volume =
- provisioner.persitenceVolumeBuilder(claim, volumeName);
-
- //remove the data which should not been compared
- V1PersistentVolume expectedVolume =
- json.getGson().fromJson(pv, V1PersistentVolume.class);
-
-
- Assert.assertEquals(expectedVolume, volume);
- }
-
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
deleted file mode 100644
index 05a632e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupOutputStream.java
+++ /dev/null
@@ -1,492 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.io;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers
- .StorageContainerException;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ChunkOutputStream;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Maintaining a list of ChunkInputStream. Write based on offset.
- *
- * Note that this may write to multiple containers in one write call. In case
- * that first container succeeded but later ones failed, the succeeded writes
- * are not rolled back.
- *
- * TODO : currently not support multi-thread access.
- */
-public class ChunkGroupOutputStream extends OutputStream {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ChunkGroupOutputStream.class);
-
- // array list's get(index) is O(1)
- private final ArrayList<ChunkOutputStreamEntry> streamEntries;
- private int currentStreamIndex;
- private long byteOffset;
- private final KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
- private final
- StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
- private final KsmKeyArgs keyArgs;
- private final int openID;
- private final XceiverClientManager xceiverClientManager;
- private final int chunkSize;
- private final String requestID;
- private boolean closed;
-
- /**
- * A constructor for testing purpose only.
- */
- @VisibleForTesting
- public ChunkGroupOutputStream() {
- streamEntries = new ArrayList<>();
- ksmClient = null;
- scmClient = null;
- keyArgs = null;
- openID = -1;
- xceiverClientManager = null;
- chunkSize = 0;
- requestID = null;
- closed = false;
- }
-
- /**
- * For testing purpose only. Not building output stream from blocks, but
- * taking from externally.
- *
- * @param outputStream
- * @param length
- */
- @VisibleForTesting
- public synchronized void addStream(OutputStream outputStream, long length) {
- streamEntries.add(new ChunkOutputStreamEntry(outputStream, length));
- }
-
- @VisibleForTesting
- public List<ChunkOutputStreamEntry> getStreamEntries() {
- return streamEntries;
- }
-
- public ChunkGroupOutputStream(
- OpenKeySession handler, XceiverClientManager xceiverClientManager,
- StorageContainerLocationProtocolClientSideTranslatorPB scmClient,
- KeySpaceManagerProtocolClientSideTranslatorPB ksmClient,
- int chunkSize, String requestId, ReplicationFactor factor,
- ReplicationType type) throws IOException {
- this.streamEntries = new ArrayList<>();
- this.currentStreamIndex = 0;
- this.byteOffset = 0;
- this.ksmClient = ksmClient;
- this.scmClient = scmClient;
- KsmKeyInfo info = handler.getKeyInfo();
- this.keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(info.getVolumeName())
- .setBucketName(info.getBucketName())
- .setKeyName(info.getKeyName())
- .setType(type)
- .setFactor(factor)
- .setDataSize(info.getDataSize()).build();
- this.openID = handler.getId();
- this.xceiverClientManager = xceiverClientManager;
- this.chunkSize = chunkSize;
- this.requestID = requestId;
- LOG.debug("Expecting open key with one block, but got" +
- info.getKeyLocationVersions().size());
- }
-
- /**
- * When a key is opened, it is possible that there are some blocks already
- * allocated to it for this open session. In this case, to make use of these
- * blocks, we need to add these blocks to stream entries. But, a key's version
- * also includes blocks from previous versions, we need to avoid adding these
- * old blocks to stream entries, because these old blocks should not be picked
- * for write. To do this, the following method checks that, only those
- * blocks created in this particular open version are added to stream entries.
- *
- * @param version the set of blocks that are pre-allocated.
- * @param openVersion the version corresponding to the pre-allocation.
- * @throws IOException
- */
- public void addPreallocateBlocks(KsmKeyLocationInfoGroup version,
- long openVersion) throws IOException {
- // server may return any number of blocks, (0 to any)
- // only the blocks allocated in this open session (block createVersion
- // equals to open session version)
- for (KsmKeyLocationInfo subKeyInfo : version.getLocationList()) {
- if (subKeyInfo.getCreateVersion() == openVersion) {
- checkKeyLocationInfo(subKeyInfo);
- }
- }
- }
-
- private void checkKeyLocationInfo(KsmKeyLocationInfo subKeyInfo)
- throws IOException {
- String containerKey = subKeyInfo.getBlockID();
- String containerName = subKeyInfo.getContainerName();
- Pipeline pipeline = scmClient.getContainer(containerName);
- XceiverClientSpi xceiverClient =
- xceiverClientManager.acquireClient(pipeline);
- // create container if needed
- if (subKeyInfo.getShouldCreateContainer()) {
- try {
- ContainerProtocolCalls.createContainer(xceiverClient, requestID);
- scmClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerName, ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.complete);
- } catch (StorageContainerException ex) {
- if (ex.getResult().equals(Result.CONTAINER_EXISTS)) {
- //container already exist, this should never happen
- LOG.debug("Container {} already exists.", containerName);
- } else {
- LOG.error("Container creation failed for {}.", containerName, ex);
- throw ex;
- }
- }
- }
- streamEntries.add(new ChunkOutputStreamEntry(containerKey,
- keyArgs.getKeyName(), xceiverClientManager, xceiverClient, requestID,
- chunkSize, subKeyInfo.getLength()));
- }
-
-
- @VisibleForTesting
- public long getByteOffset() {
- return byteOffset;
- }
-
-
- @Override
- public synchronized void write(int b) throws IOException {
- checkNotClosed();
-
- if (streamEntries.size() <= currentStreamIndex) {
- Preconditions.checkNotNull(ksmClient);
- // allocate a new block, if a exception happens, log an error and
- // throw exception to the caller directly, and the write fails.
- try {
- allocateNewBlock(currentStreamIndex);
- } catch (IOException ioe) {
- LOG.error("Allocate block fail when writing.");
- throw ioe;
- }
- }
- ChunkOutputStreamEntry entry = streamEntries.get(currentStreamIndex);
- entry.write(b);
- if (entry.getRemaining() <= 0) {
- currentStreamIndex += 1;
- }
- byteOffset += 1;
- }
-
- /**
- * Try to write the bytes sequence b[off:off+len) to streams.
- *
- * NOTE: Throws exception if the data could not fit into the remaining space.
- * In which case nothing will be written.
- * TODO:May need to revisit this behaviour.
- *
- * @param b byte data
- * @param off starting offset
- * @param len length to write
- * @throws IOException
- */
- @Override
- public synchronized void write(byte[] b, int off, int len)
- throws IOException {
- checkNotClosed();
-
- if (b == null) {
- throw new NullPointerException();
- }
- if ((off < 0) || (off > b.length) || (len < 0) ||
- ((off + len) > b.length) || ((off + len) < 0)) {
- throw new IndexOutOfBoundsException();
- }
- if (len == 0) {
- return;
- }
- int succeededAllocates = 0;
- while (len > 0) {
- if (streamEntries.size() <= currentStreamIndex) {
- Preconditions.checkNotNull(ksmClient);
- // allocate a new block, if a exception happens, log an error and
- // throw exception to the caller directly, and the write fails.
- try {
- allocateNewBlock(currentStreamIndex);
- succeededAllocates += 1;
- } catch (IOException ioe) {
- LOG.error("Try to allocate more blocks for write failed, already " +
- "allocated " + succeededAllocates + " blocks for this write.");
- throw ioe;
- }
- }
- // in theory, this condition should never violate due the check above
- // still do a sanity check.
- Preconditions.checkArgument(currentStreamIndex < streamEntries.size());
- ChunkOutputStreamEntry current = streamEntries.get(currentStreamIndex);
- int writeLen = Math.min(len, (int)current.getRemaining());
- current.write(b, off, writeLen);
- if (current.getRemaining() <= 0) {
- currentStreamIndex += 1;
- }
- len -= writeLen;
- off += writeLen;
- byteOffset += writeLen;
- }
- }
-
- /**
- * Contact KSM to get a new block. Set the new block with the index (e.g.
- * first block has index = 0, second has index = 1 etc.)
- *
- * The returned block is made to new ChunkOutputStreamEntry to write.
- *
- * @param index the index of the block.
- * @throws IOException
- */
- private void allocateNewBlock(int index) throws IOException {
- KsmKeyLocationInfo subKeyInfo = ksmClient.allocateBlock(keyArgs, openID);
- checkKeyLocationInfo(subKeyInfo);
- }
-
- @Override
- public synchronized void flush() throws IOException {
- checkNotClosed();
- for (int i = 0; i <= currentStreamIndex; i++) {
- streamEntries.get(i).flush();
- }
- }
-
- /**
- * Commit the key to KSM, this will add the blocks as the new key blocks.
- *
- * @throws IOException
- */
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
- for (ChunkOutputStreamEntry entry : streamEntries) {
- if (entry != null) {
- entry.close();
- }
- }
- if (keyArgs != null) {
- // in test, this could be null
- keyArgs.setDataSize(byteOffset);
- ksmClient.commitKey(keyArgs, openID);
- } else {
- LOG.warn("Closing ChunkGroupOutputStream, but key args is null");
- }
- }
-
- /**
- * Builder class of ChunkGroupOutputStream.
- */
- public static class Builder {
- private OpenKeySession openHandler;
- private XceiverClientManager xceiverManager;
- private StorageContainerLocationProtocolClientSideTranslatorPB scmClient;
- private KeySpaceManagerProtocolClientSideTranslatorPB ksmClient;
- private int chunkSize;
- private String requestID;
- private ReplicationType type;
- private ReplicationFactor factor;
-
- public Builder setHandler(OpenKeySession handler) {
- this.openHandler = handler;
- return this;
- }
-
- public Builder setXceiverClientManager(XceiverClientManager manager) {
- this.xceiverManager = manager;
- return this;
- }
-
- public Builder setScmClient(
- StorageContainerLocationProtocolClientSideTranslatorPB client) {
- this.scmClient = client;
- return this;
- }
-
- public Builder setKsmClient(
- KeySpaceManagerProtocolClientSideTranslatorPB client) {
- this.ksmClient = client;
- return this;
- }
-
- public Builder setChunkSize(int size) {
- this.chunkSize = size;
- return this;
- }
-
- public Builder setRequestID(String id) {
- this.requestID = id;
- return this;
- }
-
- public Builder setType(ReplicationType replicationType) {
- this.type = replicationType;
- return this;
- }
-
- public Builder setFactor(ReplicationFactor replicationFactor) {
- this.factor = replicationFactor;
- return this;
- }
-
- public ChunkGroupOutputStream build() throws IOException {
- return new ChunkGroupOutputStream(openHandler, xceiverManager, scmClient,
- ksmClient, chunkSize, requestID, factor, type);
- }
- }
-
- private static class ChunkOutputStreamEntry extends OutputStream {
- private OutputStream outputStream;
- private final String containerKey;
- private final String key;
- private final XceiverClientManager xceiverClientManager;
- private final XceiverClientSpi xceiverClient;
- private final String requestId;
- private final int chunkSize;
- // total number of bytes that should be written to this stream
- private final long length;
- // the current position of this stream 0 <= currentPosition < length
- private long currentPosition;
-
- ChunkOutputStreamEntry(String containerKey, String key,
- XceiverClientManager xceiverClientManager,
- XceiverClientSpi xceiverClient, String requestId, int chunkSize,
- long length) {
- this.outputStream = null;
- this.containerKey = containerKey;
- this.key = key;
- this.xceiverClientManager = xceiverClientManager;
- this.xceiverClient = xceiverClient;
- this.requestId = requestId;
- this.chunkSize = chunkSize;
-
- this.length = length;
- this.currentPosition = 0;
- }
-
- /**
- * For testing purpose, taking a some random created stream instance.
- * @param outputStream a existing writable output stream
- * @param length the length of data to write to the stream
- */
- ChunkOutputStreamEntry(OutputStream outputStream, long length) {
- this.outputStream = outputStream;
- this.containerKey = null;
- this.key = null;
- this.xceiverClientManager = null;
- this.xceiverClient = null;
- this.requestId = null;
- this.chunkSize = -1;
-
- this.length = length;
- this.currentPosition = 0;
- }
-
- long getLength() {
- return length;
- }
-
- long getRemaining() {
- return length - currentPosition;
- }
-
- private synchronized void checkStream() {
- if (this.outputStream == null) {
- this.outputStream = new ChunkOutputStream(containerKey,
- key, xceiverClientManager, xceiverClient,
- requestId, chunkSize);
- }
- }
-
- @Override
- public void write(int b) throws IOException {
- checkStream();
- outputStream.write(b);
- this.currentPosition += 1;
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- checkStream();
- outputStream.write(b, off, len);
- this.currentPosition += len;
- }
-
- @Override
- public void flush() throws IOException {
- if (this.outputStream != null) {
- this.outputStream.flush();
- }
- }
-
- @Override
- public void close() throws IOException {
- if (this.outputStream != null) {
- this.outputStream.close();
- }
- }
- }
-
- /**
- * Verify that the output stream is open. Non blocking; this gives
- * the last state of the volatile {@link #closed} field.
- * @throws IOException if the connection is closed.
- */
- private void checkNotClosed() throws IOException {
- if (closed) {
- throw new IOException(
- ": " + FSExceptionMessages.STREAM_IS_CLOSED + " Key: " + keyArgs
- .getKeyName());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
deleted file mode 100644
index baf1887..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/LengthInputStream.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import java.io.FilterInputStream;
-import java.io.InputStream;
-
-/**
- * An input stream with length.
- */
-public class LengthInputStream extends FilterInputStream {
-
- private final long length;
-
- /**
- * Create an stream.
- * @param in the underlying input stream.
- * @param length the length of the stream.
- */
- public LengthInputStream(InputStream in, long length) {
- super(in);
- this.length = length;
- }
-
- /** @return the length. */
- public long getLength() {
- return length;
- }
-
- public InputStream getWrappedStream() {
- return in;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
deleted file mode 100644
index ca6f7aa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneContainerTranslation.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-
-/**
- * This class contains methods that define the translation between the Ozone
- * domain model and the storage container domain model.
- */
-final class OzoneContainerTranslation {
-
- /**
- * Creates key data intended for reading a container key.
- *
- * @param containerName container name
- * @param containerKey container key
- * @return KeyData intended for reading the container key
- */
- public static KeyData containerKeyDataForRead(String containerName,
- String containerKey) {
- return KeyData
- .newBuilder()
- .setContainerName(containerName)
- .setName(containerKey)
- .build();
- }
-
- /**
- * There is no need to instantiate this class.
- */
- private OzoneContainerTranslation() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
deleted file mode 100644
index 3857bd0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneInputStream.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import org.apache.hadoop.scm.storage.ChunkInputStream;
-
-import java.io.IOException;
-import java.io.InputStream;
-
-/**
- * OzoneInputStream is used to read data from Ozone.
- * It uses SCM's {@link ChunkInputStream} for reading the data.
- */
-public class OzoneInputStream extends InputStream {
-
- private final InputStream inputStream;
-
- /**
- * Constructs OzoneInputStream with ChunkInputStream.
- *
- * @param inputStream
- */
- public OzoneInputStream(InputStream inputStream) {
- this.inputStream = inputStream;
- }
-
- @Override
- public int read() throws IOException {
- return inputStream.read();
- }
-
- @Override
- public synchronized void close() throws IOException {
- inputStream.close();
- }
-
- @Override
- public int available() throws IOException {
- return inputStream.available();
- }
-
- public InputStream getInputStream() {
- return inputStream;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
deleted file mode 100644
index 5369220..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/OzoneOutputStream.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * OzoneOutputStream is used to write data into Ozone.
- * It uses SCM's {@link ChunkGroupOutputStream} for writing the data.
- */
-public class OzoneOutputStream extends OutputStream {
-
- private final OutputStream outputStream;
-
- /**
- * Constructs OzoneOutputStream with ChunkGroupOutputStream.
- *
- * @param outputStream
- */
- public OzoneOutputStream(OutputStream outputStream) {
- this.outputStream = outputStream;
- }
-
- @Override
- public void write(int b) throws IOException {
- outputStream.write(b);
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- outputStream.write(b, off, len);
- }
-
- @Override
- public synchronized void flush() throws IOException {
- outputStream.flush();
- }
-
- @Override
- public synchronized void close() throws IOException {
- //commitKey can be done here, if needed.
- outputStream.close();
- }
-
- public OutputStream getOutputStream() {
- return outputStream;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
deleted file mode 100644
index 493ece8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.io;
-
-/**
- * This package contains Ozone I/O classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
deleted file mode 100644
index 7e2591a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * This package contains Ozone Client classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
deleted file mode 100644
index 64a970e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java
+++ /dev/null
@@ -1,323 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.protocol;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * An implementer of this interface is capable of connecting to Ozone Cluster
- * and perform client operations. The protocol used for communication is
- * determined by the implementation class specified by
- * property <code>ozone.client.protocol</code>. The build-in implementation
- * includes: {@link org.apache.hadoop.ozone.client.rpc.RpcClient} for RPC and
- * {@link org.apache.hadoop.ozone.client.rest.RestClient} for REST.
- */
-public interface ClientProtocol {
-
- /**
- * Creates a new Volume.
- * @param volumeName Name of the Volume
- * @throws IOException
- */
- void createVolume(String volumeName)
- throws IOException;
-
- /**
- * Creates a new Volume with properties set in VolumeArgs.
- * @param volumeName Name of the Volume
- * @param args Properties to be set for the Volume
- * @throws IOException
- */
- void createVolume(String volumeName, VolumeArgs args)
- throws IOException;
-
- /**
- * Sets the owner of volume.
- * @param volumeName Name of the Volume
- * @param owner to be set for the Volume
- * @throws IOException
- */
- void setVolumeOwner(String volumeName, String owner) throws IOException;
-
- /**
- * Set Volume Quota.
- * @param volumeName Name of the Volume
- * @param quota Quota to be set for the Volume
- * @throws IOException
- */
- void setVolumeQuota(String volumeName, OzoneQuota quota)
- throws IOException;
-
- /**
- * Returns {@link OzoneVolume}.
- * @param volumeName Name of the Volume
- * @return {@link OzoneVolume}
- * @throws IOException
- * */
- OzoneVolume getVolumeDetails(String volumeName)
- throws IOException;
-
- /**
- * Checks if a Volume exists and the user with a role specified has access
- * to the Volume.
- * @param volumeName Name of the Volume
- * @param acl requested acls which needs to be checked for access
- * @return Boolean - True if the user with a role can access the volume.
- * This is possible for owners of the volume and admin users
- * @throws IOException
- */
- boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
- throws IOException;
-
- /**
- * Deletes an empty Volume.
- * @param volumeName Name of the Volume
- * @throws IOException
- */
- void deleteVolume(String volumeName) throws IOException;
-
- /**
- * Lists all volumes in the cluster that matches the volumePrefix,
- * size of the returned list depends on maxListResult. If volume prefix
- * is null, returns all the volumes. The caller has to make multiple calls
- * to read all volumes.
- *
- * @param volumePrefix Volume prefix to match
- * @param prevVolume Starting point of the list, this volume is excluded
- * @param maxListResult Max number of volumes to return.
- * @return {@code List<OzoneVolume>}
- * @throws IOException
- */
- List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
- int maxListResult)
- throws IOException;
-
- /**
- * Lists all volumes in the cluster that are owned by the specified
- * user and matches the volumePrefix, size of the returned list depends on
- * maxListResult. If the user is null, return volumes owned by current user.
- * If volume prefix is null, returns all the volumes. The caller has to make
- * multiple calls to read all volumes.
- *
- * @param user User Name
- * @param volumePrefix Volume prefix to match
- * @param prevVolume Starting point of the list, this volume is excluded
- * @param maxListResult Max number of volumes to return.
- * @return {@code List<OzoneVolume>}
- * @throws IOException
- */
- List<OzoneVolume> listVolumes(String user, String volumePrefix,
- String prevVolume, int maxListResult)
- throws IOException;
-
- /**
- * Creates a new Bucket in the Volume.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @throws IOException
- */
- void createBucket(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * Creates a new Bucket in the Volume, with properties set in BucketArgs.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param bucketArgs Bucket Arguments
- * @throws IOException
- */
- void createBucket(String volumeName, String bucketName,
- BucketArgs bucketArgs)
- throws IOException;
-
- /**
- * Adds ACLs to the Bucket.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param addAcls ACLs to be added
- * @throws IOException
- */
- void addBucketAcls(String volumeName, String bucketName,
- List<OzoneAcl> addAcls)
- throws IOException;
-
- /**
- * Removes ACLs from a Bucket.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param removeAcls ACLs to be removed
- * @throws IOException
- */
- void removeBucketAcls(String volumeName, String bucketName,
- List<OzoneAcl> removeAcls)
- throws IOException;
-
-
- /**
- * Enables or disables Bucket Versioning.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param versioning True to enable Versioning, False to disable.
- * @throws IOException
- */
- void setBucketVersioning(String volumeName, String bucketName,
- Boolean versioning)
- throws IOException;
-
- /**
- * Sets the Storage Class of a Bucket.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param storageType StorageType to be set
- * @throws IOException
- */
- void setBucketStorageType(String volumeName, String bucketName,
- StorageType storageType)
- throws IOException;
-
- /**
- * Deletes a bucket if it is empty.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @throws IOException
- */
- void deleteBucket(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * True if the bucket exists and user has read access
- * to the bucket else throws Exception.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @throws IOException
- */
- void checkBucketAccess(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * Returns {@link OzoneBucket}.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @return {@link OzoneBucket}
- * @throws IOException
- */
- OzoneBucket getBucketDetails(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * Returns the List of Buckets in the Volume that matches the bucketPrefix,
- * size of the returned list depends on maxListResult. The caller has to make
- * multiple calls to read all volumes.
- * @param volumeName Name of the Volume
- * @param bucketPrefix Bucket prefix to match
- * @param prevBucket Starting point of the list, this bucket is excluded
- * @param maxListResult Max number of buckets to return.
- * @return {@code List<OzoneBucket>}
- * @throws IOException
- */
- List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException;
-
- /**
- * Writes a key in an existing bucket.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Name of the Key
- * @param size Size of the data
- * @return {@link OzoneOutputStream}
- *
- */
- OzoneOutputStream createKey(String volumeName, String bucketName,
- String keyName, long size, ReplicationType type,
- ReplicationFactor factor)
- throws IOException;
-
- /**
- * Reads a key from an existing bucket.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Name of the Key
- * @return {@link OzoneInputStream}
- * @throws IOException
- */
- OzoneInputStream getKey(String volumeName, String bucketName, String keyName)
- throws IOException;
-
-
- /**
- * Deletes an existing key.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Name of the Key
- * @throws IOException
- */
- void deleteKey(String volumeName, String bucketName, String keyName)
- throws IOException;
-
-
- /**
- * Returns list of Keys in {Volume/Bucket} that matches the keyPrefix,
- * size of the returned list depends on maxListResult. The caller has
- * to make multiple calls to read all keys.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyPrefix Bucket prefix to match
- * @param prevKey Starting point of the list, this key is excluded
- * @param maxListResult Max number of buckets to return.
- * @return {@code List<OzoneKey>}
- * @throws IOException
- */
- List<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix, String prevKey, int maxListResult)
- throws IOException;
-
-
- /**
- * Get OzoneKey.
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Key name
- * @return {@link OzoneKey}
- * @throws IOException
- */
- OzoneKey getKeyDetails(String volumeName, String bucketName,
- String keyName)
- throws IOException;
-
- /**
- * Close and release the resources.
- */
- void close() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
deleted file mode 100644
index f4890a1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/protocol/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.protocol;
-
-/**
- * This package contains Ozone client protocol library classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
deleted file mode 100644
index 93b3417..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/DefaultRestServerSelector.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-
-import java.util.List;
-import java.util.Random;
-
-/**
- * Default selector randomly picks one of the REST Server from the list.
- */
-public class DefaultRestServerSelector implements RestServerSelector {
-
- @Override
- public ServiceInfo getRestServer(List<ServiceInfo> restServices) {
- return restServices.get(
- new Random().nextInt(restServices.size()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
deleted file mode 100644
index 953e399..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneException.java
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-
-/**
- * Class the represents various errors returned by the
- * Ozone Layer.
- */
-@InterfaceAudience.Private
-public class OzoneException extends Exception {
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(OzoneException.class);
- private static final ObjectMapper MAPPER;
-
- static {
- MAPPER = new ObjectMapper();
- MAPPER.setVisibility(
- MAPPER.getSerializationConfig().getDefaultVisibilityChecker()
- .withCreatorVisibility(JsonAutoDetect.Visibility.NONE)
- .withFieldVisibility(JsonAutoDetect.Visibility.NONE)
- .withGetterVisibility(JsonAutoDetect.Visibility.NONE)
- .withIsGetterVisibility(JsonAutoDetect.Visibility.NONE)
- .withSetterVisibility(JsonAutoDetect.Visibility.NONE));
- }
-
- @JsonProperty("httpCode")
- private long httpCode;
- @JsonProperty("shortMessage")
- private String shortMessage;
- @JsonProperty("resource")
- private String resource;
- @JsonProperty("message")
- private String message;
- @JsonProperty("requestID")
- private String requestId;
- @JsonProperty("hostName")
- private String hostID;
-
- /**
- * Constructs a new exception with {@code null} as its detail message. The
- * cause is not initialized, and may subsequently be initialized by a call
- * to {@link #initCause}.
- *
- * This constructor is needed by Json Serializer.
- */
- public OzoneException() {
- }
-
-
- /**
- * Constructor that allows a shortMessage and exception.
- *
- * @param httpCode Error Code
- * @param shortMessage Short Message
- * @param ex Exception
- */
- public OzoneException(long httpCode, String shortMessage, Exception ex) {
- super(ex);
- this.message = ex.getMessage();
- this.shortMessage = shortMessage;
- this.httpCode = httpCode;
- }
-
-
- /**
- * Constructor that allows a shortMessage.
- *
- * @param httpCode Error Code
- * @param shortMessage Short Message
- */
- public OzoneException(long httpCode, String shortMessage) {
- this.shortMessage = shortMessage;
- this.httpCode = httpCode;
- }
-
- /**
- * Constructor that allows a shortMessage and long message.
- *
- * @param httpCode Error Code
- * @param shortMessage Short Message
- * @param message long error message
- */
- public OzoneException(long httpCode, String shortMessage, String message) {
- this.shortMessage = shortMessage;
- this.message = message;
- this.httpCode = httpCode;
- }
-
- /**
- * Constructor that allows a shortMessage, a long message and an exception.
- *
- * @param httpCode Error code
- * @param shortMessage Short message
- * @param message Long error message
- * @param ex Exception
- */
- public OzoneException(long httpCode, String shortMessage,
- String message, Exception ex) {
- super(ex);
- this.shortMessage = shortMessage;
- this.message = message;
- this.httpCode = httpCode;
- }
-
- /**
- * Returns the Resource that was involved in the stackTraceString.
- *
- * @return String
- */
- public String getResource() {
- return resource;
- }
-
- /**
- * Sets Resource.
- *
- * @param resourceName - Name of the Resource
- */
- public void setResource(String resourceName) {
- this.resource = resourceName;
- }
-
- /**
- * Gets a detailed message for the error.
- *
- * @return String
- */
- public String getMessage() {
- return message;
- }
-
- /**
- * Sets the error message.
- *
- * @param longMessage - Long message
- */
- public void setMessage(String longMessage) {
- this.message = longMessage;
- }
-
- /**
- * Returns request Id.
- *
- * @return String
- */
- public String getRequestId() {
- return requestId;
- }
-
- /**
- * Sets request ID.
- *
- * @param ozoneRequestId Request ID generated by the Server
- */
- public void setRequestId(String ozoneRequestId) {
- this.requestId = ozoneRequestId;
- }
-
- /**
- * Returns short error string.
- *
- * @return String
- */
- public String getShortMessage() {
- return shortMessage;
- }
-
- /**
- * Sets short error string.
- *
- * @param shortError Short Error Code
- */
- public void setShortMessage(String shortError) {
- this.shortMessage = shortError;
- }
-
- /**
- * Returns hostID.
- *
- * @return String
- */
- public String getHostID() {
- return hostID;
- }
-
- /**
- * Sets host ID.
- *
- * @param hostName host Name
- */
- public void setHostID(String hostName) {
- this.hostID = hostName;
- }
-
- /**
- * Returns http error code.
- *
- * @return long
- */
- public long getHttpCode() {
- return httpCode;
- }
-
- /**
- * Sets http status.
- *
- * @param httpStatus http error code.
- */
- public void setHttpCode(long httpStatus) {
- this.httpCode = httpStatus;
- }
-
- /**
- * Returns a Json String.
- *
- * @return JSON representation of the Error
- */
- public String toJsonString() {
- try {
- return MAPPER.writeValueAsString(this);
- } catch (IOException ex) {
- // TODO : Log this error on server side.
- }
- // TODO : Replace this with a JSON Object -- That represents this error.
- return "500 Internal Server Error";
- }
-
- /**
- * Parses an Exception record.
- *
- * @param jsonString - Exception in Json format.
- *
- * @return OzoneException Object
- *
- * @throws IOException
- */
- public static OzoneException parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
deleted file mode 100644
index 6c479f7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/OzoneExceptionMapper.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-
-import javax.ws.rs.core.Response;
-import javax.ws.rs.ext.ExceptionMapper;
-
-import org.slf4j.MDC;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Class the represents various errors returned by the
- * Object Layer.
- */
-public class OzoneExceptionMapper implements ExceptionMapper<OzoneException> {
- private static final Logger LOG =
- LoggerFactory.getLogger(OzoneExceptionMapper.class);
-
- @Override
- public Response toResponse(OzoneException exception) {
- LOG.debug("Returning exception. ex: {}", exception.toJsonString());
- MDC.clear();
- return Response.status((int)exception.getHttpCode())
- .entity(exception.toJsonString()).build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
deleted file mode 100644
index c420ce5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestClient.java
+++ /dev/null
@@ -1,799 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import com.fasterxml.jackson.core.type.TypeReference;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.client.rest.response.BucketInfo;
-import org.apache.hadoop.ozone.client.rest.response.KeyInfo;
-import org.apache.hadoop.ozone.client.rest.response.VolumeInfo;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Time;
-import org.apache.http.HttpEntity;
-import org.apache.http.HttpHeaders;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpDelete;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.client.methods.HttpPost;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.client.methods.HttpUriRequest;
-import org.apache.http.client.utils.URIBuilder;
-import org.apache.http.entity.InputStreamEntity;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
-import org.apache.http.util.EntityUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.text.ParseException;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.FutureTask;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static java.net.HttpURLConnection.HTTP_CREATED;
-import static java.net.HttpURLConnection.HTTP_OK;
-
-/**
- * Ozone Client REST protocol implementation. It uses REST protocol to
- * connect to Ozone Handler that executes client calls.
- */
-public class RestClient implements ClientProtocol {
-
- private static final String PATH_SEPARATOR = "/";
- private static final Logger LOG = LoggerFactory.getLogger(RpcClient.class);
-
- private final Configuration conf;
- private final URI ozoneRestUri;
- private final CloseableHttpClient httpClient;
- private final UserGroupInformation ugi;
- private final OzoneAcl.OzoneACLRights userRights;
-
- /**
- * Creates RestClient instance with the given configuration.
- * @param conf Configuration
- * @throws IOException
- */
- public RestClient(Configuration conf)
- throws IOException {
- try {
- Preconditions.checkNotNull(conf);
- this.conf = conf;
-
- long socketTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- long connectionTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- int maxConnection = conf.getInt(
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_MAX,
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT);
-
- int maxConnectionPerRoute = conf.getInt(
- OzoneConfigKeys.OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX,
- OzoneConfigKeys
- .OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT
- );
-
- /*
- To make RestClient Thread safe, creating the HttpClient with
- ThreadSafeClientConnManager.
- */
- PoolingHttpClientConnectionManager connManager =
- new PoolingHttpClientConnectionManager();
- connManager.setMaxTotal(maxConnection);
- connManager.setDefaultMaxPerRoute(maxConnectionPerRoute);
-
- this.httpClient = HttpClients.custom()
- .setConnectionManager(connManager)
- .setDefaultRequestConfig(
- RequestConfig.custom()
- .setSocketTimeout(Math.toIntExact(socketTimeout))
- .setConnectTimeout(Math.toIntExact(connectionTimeout))
- .build())
- .build();
- this.ugi = UserGroupInformation.getCurrentUser();
- this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
- KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
-
- // TODO: Add new configuration parameter to configure RestServerSelector.
- RestServerSelector defaultSelector = new DefaultRestServerSelector();
- InetSocketAddress restServer = getOzoneRestServerAddress(defaultSelector);
- URIBuilder uriBuilder = new URIBuilder()
- .setScheme("http")
- .setHost(restServer.getHostName())
- .setPort(restServer.getPort());
- this.ozoneRestUri = uriBuilder.build();
-
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- private InetSocketAddress getOzoneRestServerAddress(
- RestServerSelector selector) throws IOException {
- String httpAddress = conf.get(KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY);
-
- if (httpAddress == null) {
- throw new IllegalArgumentException(
- KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY + " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration for" +
- " details on configuring Ozone.");
- }
-
- HttpGet httpGet = new HttpGet("http://" + httpAddress + "/serviceList");
- HttpEntity entity = executeHttpRequest(httpGet);
- try {
- String serviceListJson = EntityUtils.toString(entity);
-
- ObjectMapper objectMapper = new ObjectMapper();
- TypeReference<List<ServiceInfo>> serviceInfoReference =
- new TypeReference<List<ServiceInfo>>() {
- };
- List<ServiceInfo> services = objectMapper.readValue(
- serviceListJson, serviceInfoReference);
-
- List<ServiceInfo> dataNodeInfos = services.stream().filter(
- a -> a.getNodeType().equals(OzoneProtos.NodeType.DATANODE))
- .collect(Collectors.toList());
-
- ServiceInfo restServer = selector.getRestServer(dataNodeInfos);
-
- return NetUtils.createSocketAddr(restServer.getHostname() + ":" +
- restServer.getPort(ServicePort.Type.HTTP));
- } finally {
- EntityUtils.consume(entity);
- }
- }
-
- @Override
- public void createVolume(String volumeName) throws IOException {
- createVolume(volumeName, VolumeArgs.newBuilder().build());
- }
-
- @Override
- public void createVolume(String volumeName, VolumeArgs volArgs)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- String owner = volArgs.getOwner() == null ?
- ugi.getUserName() : volArgs.getOwner();
- //TODO: support for ACLs has to be done in OzoneHandler (rest server)
- /**
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- owner, userRights));
- //ACLs from VolumeArgs
- if(volArgs.getAcls() != null) {
- listOfAcls.addAll(volArgs.getAcls());
- }
- */
- builder.setPath(PATH_SEPARATOR + volumeName);
-
- String quota = volArgs.getQuota();
- if(quota != null) {
- builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quota);
- }
-
- HttpPost httpPost = new HttpPost(builder.build());
- addOzoneHeaders(httpPost);
- //use admin from VolumeArgs, if it's present
- if(volArgs.getAdmin() != null) {
- httpPost.removeHeaders(HttpHeaders.AUTHORIZATION);
- httpPost.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- volArgs.getAdmin());
- }
- httpPost.addHeader(Header.OZONE_USER, owner);
- LOG.info("Creating Volume: {}, with {} as owner and quota set to {}.",
- volumeName, owner, quota == null ? "default" : quota);
- EntityUtils.consume(executeHttpRequest(httpPost));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
-
- @Override
- public void setVolumeOwner(String volumeName, String owner)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(owner);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
- httpPut.addHeader(Header.OZONE_USER, owner);
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setVolumeQuota(String volumeName, OzoneQuota quota)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(quota);
- String quotaString = quota.toString();
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- builder.setParameter(Header.OZONE_QUOTA_QUERY_TAG, quotaString);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public OzoneVolume getVolumeDetails(String volumeName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_VOLUME);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- VolumeInfo volInfo =
- VolumeInfo.parse(EntityUtils.toString(response));
- //TODO: OzoneHandler in datanode has to be modified to send ACLs
- OzoneVolume volume = new OzoneVolume(conf,
- this,
- volInfo.getVolumeName(),
- volInfo.getCreatedBy(),
- volInfo.getOwner().getName(),
- volInfo.getQuota().sizeInBytes(),
- OzoneClientUtils.formatDateTime(volInfo.getCreatedOn()),
- null);
- EntityUtils.consume(response);
- return volume;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void deleteVolume(String volumeName) throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String volumePrefix, String prevKey,
- int maxListResult)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String user, String volumePrefix,
- String prevKey, int maxListResult)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void createBucket(String volumeName, String bucketName)
- throws IOException {
- createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
- }
-
- @Override
- public void createBucket(
- String volumeName, String bucketName, BucketArgs bucketArgs)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(bucketArgs);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- OzoneConsts.Versioning versioning = OzoneConsts.Versioning.DISABLED;
- if(bucketArgs.getVersioning() != null &&
- bucketArgs.getVersioning()) {
- versioning = OzoneConsts.Versioning.ENABLED;
- }
- StorageType storageType = bucketArgs.getStorageType() == null ?
- StorageType.DEFAULT : bucketArgs.getStorageType();
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPost httpPost = new HttpPost(builder.build());
- addOzoneHeaders(httpPost);
-
- //ACLs from BucketArgs
- if(bucketArgs.getAcls() != null) {
- for (OzoneAcl acl : bucketArgs.getAcls()) {
- httpPost.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
- }
- }
- httpPost.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
- httpPost.addHeader(Header.OZONE_BUCKET_VERSIONING,
- versioning.toString());
- LOG.info("Creating Bucket: {}/{}, with Versioning {} and Storage Type" +
- " set to {}", volumeName, bucketName, versioning,
- storageType);
-
- EntityUtils.consume(executeHttpRequest(httpPost));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void addBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> addAcls)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(addAcls);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- for (OzoneAcl acl : addAcls) {
- httpPut.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_ADD + " " + acl.toString());
- }
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void removeBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> removeAcls)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(removeAcls);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- for (OzoneAcl acl : removeAcls) {
- httpPut.addHeader(
- Header.OZONE_ACLS, Header.OZONE_ACL_REMOVE + " " + acl.toString());
- }
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setBucketVersioning(
- String volumeName, String bucketName, Boolean versioning)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(versioning);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- httpPut.addHeader(Header.OZONE_BUCKET_VERSIONING,
- getBucketVersioning(versioning).toString());
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void setBucketStorageType(
- String volumeName, String bucketName, StorageType storageType)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(storageType);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
-
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpPut httpPut = new HttpPut(builder.build());
- addOzoneHeaders(httpPut);
-
- httpPut.addHeader(Header.OZONE_STORAGE_TYPE, storageType.toString());
- EntityUtils.consume(executeHttpRequest(httpPut));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void deleteBucket(String volumeName, String bucketName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void checkBucketAccess(String volumeName, String bucketName)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public OzoneBucket getBucketDetails(String volumeName, String bucketName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_BUCKET);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- BucketInfo bucketInfo =
- BucketInfo.parse(EntityUtils.toString(response));
- OzoneBucket bucket = new OzoneBucket(conf,
- this,
- bucketInfo.getVolumeName(),
- bucketInfo.getBucketName(),
- bucketInfo.getAcls(),
- bucketInfo.getStorageType(),
- getBucketVersioningFlag(bucketInfo.getVersioning()),
- OzoneClientUtils.formatDateTime(bucketInfo.getCreatedOn()));
- EntityUtils.consume(response);
- return bucket;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- /**
- * Writes a key in an existing bucket.
- *
- * @param volumeName Name of the Volume
- * @param bucketName Name of the Bucket
- * @param keyName Name of the Key
- * @param size Size of the data
- * @param type
- * @param factor @return {@link OzoneOutputStream}
- */
- @Override
- public OzoneOutputStream createKey(
- String volumeName, String bucketName, String keyName, long size,
- ReplicationType type, ReplicationFactor factor)
- throws IOException {
- // TODO: Once ReplicationType and ReplicationFactor are supported in
- // OzoneHandler (in Datanode), set them in header.
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName +
- PATH_SEPARATOR + keyName);
- HttpPut putRequest = new HttpPut(builder.build());
- addOzoneHeaders(putRequest);
- PipedInputStream in = new PipedInputStream();
- OutputStream out = new PipedOutputStream(in);
- putRequest.setEntity(new InputStreamEntity(in, size));
- FutureTask<HttpEntity> futureTask =
- new FutureTask<>(() -> executeHttpRequest(putRequest));
- new Thread(futureTask).start();
- OzoneOutputStream outputStream = new OzoneOutputStream(
- new OutputStream() {
- @Override
- public void write(int b) throws IOException {
- out.write(b);
- }
-
- @Override
- public void close() throws IOException {
- try {
- out.close();
- EntityUtils.consume(futureTask.get());
- } catch (ExecutionException | InterruptedException e) {
- throw new IOException(e);
- }
- }
- });
-
- return outputStream;
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public OzoneInputStream getKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName +
- PATH_SEPARATOR + keyName);
- HttpGet getRequest = new HttpGet(builder.build());
- addOzoneHeaders(getRequest);
- HttpEntity entity = executeHttpRequest(getRequest);
- PipedInputStream in = new PipedInputStream();
- OutputStream out = new PipedOutputStream(in);
- FutureTask<Void> futureTask =
- new FutureTask<>(() -> {
- entity.writeTo(out);
- out.close();
- return null;
- });
- new Thread(futureTask).start();
- OzoneInputStream inputStream = new OzoneInputStream(
- new InputStream() {
-
- @Override
- public int read() throws IOException {
- return in.read();
- }
-
- @Override
- public void close() throws IOException {
- in.close();
- EntityUtils.consume(entity);
- }
- });
-
- return inputStream;
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public void deleteKey(String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
- HttpDelete httpDelete = new HttpDelete(builder.build());
- addOzoneHeaders(httpDelete);
- EntityUtils.consume(executeHttpRequest(httpDelete));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
- }
-
- @Override
- public List<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix, String prevKey,
- int maxListResult)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public OzoneKey getKeyDetails(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- try {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- URIBuilder builder = new URIBuilder(ozoneRestUri);
- builder.setPath(PATH_SEPARATOR + volumeName +
- PATH_SEPARATOR + bucketName + PATH_SEPARATOR + keyName);
- builder.setParameter(Header.OZONE_INFO_QUERY_TAG,
- Header.OZONE_INFO_QUERY_KEY);
- HttpGet httpGet = new HttpGet(builder.build());
- addOzoneHeaders(httpGet);
- HttpEntity response = executeHttpRequest(httpGet);
- KeyInfo keyInfo =
- KeyInfo.parse(EntityUtils.toString(response));
- OzoneKey key = new OzoneKey(volumeName,
- bucketName,
- keyInfo.getKeyName(),
- keyInfo.getSize(),
- OzoneClientUtils.formatDateTime(keyInfo.getCreatedOn()),
- OzoneClientUtils.formatDateTime(keyInfo.getModifiedOn()));
- EntityUtils.consume(response);
- return key;
- } catch (URISyntaxException | ParseException e) {
- throw new IOException(e);
- }
- }
-
- /**
- * Adds Ozone headers to http request.
- *
- * @param httpRequest Http Request
- */
- private void addOzoneHeaders(HttpUriRequest httpRequest) {
- httpRequest.addHeader(HttpHeaders.AUTHORIZATION,
- Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME + " " +
- ugi.getUserName());
- httpRequest.addHeader(HttpHeaders.DATE,
- OzoneClientUtils.formatDateTime(Time.monotonicNow()));
- httpRequest.addHeader(Header.OZONE_VERSION_HEADER,
- Header.OZONE_V1_VERSION_HEADER);
- }
-
- /**
- * Sends the http request to server and returns the response HttpEntity.
- * It's responsibility of the caller to consume and close response HttpEntity
- * by calling {@code EntityUtils.consume}
- *
- * @param httpUriRequest http request
- * @throws IOException
- */
- private HttpEntity executeHttpRequest(HttpUriRequest httpUriRequest)
- throws IOException {
- HttpResponse response = httpClient.execute(httpUriRequest);
- int errorCode = response.getStatusLine().getStatusCode();
- HttpEntity entity = response.getEntity();
- if ((errorCode == HTTP_OK) || (errorCode == HTTP_CREATED)) {
- return entity;
- }
- if (entity != null) {
- throw new IOException(
- OzoneException.parse(EntityUtils.toString(entity)));
- } else {
- throw new IOException("Unexpected null in http payload," +
- " while processing request");
- }
- }
-
- /**
- * Converts OzoneConts.Versioning to boolean.
- *
- * @param version
- * @return corresponding boolean value
- */
- private Boolean getBucketVersioningFlag(
- OzoneConsts.Versioning version) {
- if(version != null) {
- switch(version) {
- case ENABLED:
- return true;
- case NOT_DEFINED:
- case DISABLED:
- default:
- return false;
- }
- }
- return false;
- }
-
- /**
- * Converts Bucket versioning flag into OzoneConts.Versioning.
- *
- * @param flag versioning flag
- * @return corresponding OzoneConts.Versionin
- */
- private OzoneConsts.Versioning getBucketVersioning(Boolean flag) {
- if(flag != null) {
- if(flag) {
- return OzoneConsts.Versioning.ENABLED;
- } else {
- return OzoneConsts.Versioning.DISABLED;
- }
- }
- return OzoneConsts.Versioning.NOT_DEFINED;
- }
-
- @Override
- public void close() throws IOException {
- httpClient.close();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
deleted file mode 100644
index 54e219b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/RestServerSelector.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-
-import java.util.List;
-
-/**
- * The implementor of this interface should select the REST server which will
- * be used by the client to connect to Ozone Cluster, given list of
- * REST Servers/DataNodes (DataNodes are the ones which hosts REST Service).
- */
-public interface RestServerSelector {
-
- /**
- * Returns the REST Service which will be used by the client for connection.
- *
- * @param restServices list of available REST servers
- * @return ServiceInfo
- */
- ServiceInfo getRestServer(List<ServiceInfo> restServices);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
deleted file mode 100644
index 233e788..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/exceptions/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.rest.exceptions;
-
-/**
- * This package contains ozone rest client libraries.
- */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 63f3376..0e349d3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -18,24 +18,6 @@
package org.apache.hadoop.fs;
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections.map.CaseInsensitiveMap;
-import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
-import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.io.nativeio.NativeIO;
-import org.apache.hadoop.util.Shell;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
@@ -71,8 +53,26 @@ import java.util.zip.CheckedOutputStream;
import java.util.zip.GZIPInputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
-import java.util.zip.ZipOutputStream;
import java.util.zip.ZipInputStream;
+import java.util.zip.ZipOutputStream;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.collections.map.CaseInsensitiveMap;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.nativeio.NativeIO;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.util.Shell.ShellCommandExecutor;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A collection of file-processing util methods.
@@ -91,9 +91,9 @@ public class FileUtil {
public static final int SYMLINK_NO_PRIVILEGE = 2;
/**
- * Buffer size used while zipping and unzipping zip-ed archives.
+ * Buffer size for copy the content of compressed file to new file.
*/
- private static final int BUFFER_SIZE = 8192;
+ private static final int BUFFER_SIZE = 8_192;
/**
* convert an array of FileStatus to an array of Path
@@ -613,7 +613,6 @@ public class FileUtil {
}
/**
-<<<<<<< HEAD
* creates zip archieve of the source dir and writes a zip file.
*
* @param sourceDir - The directory to zip.
@@ -673,7 +672,7 @@ public class FileUtil {
}
/**
- * Given a File input it will unzip the file in a the unzip directory
+ * Given a stream input it will unzip the it in the unzip directory.
* passed as the second parameter
* @param inputStream The zip file as input
* @param toDir The unzip directory where to unzip the zip file.
@@ -731,12 +730,12 @@ public class FileUtil {
if (!file.getParentFile().mkdirs()) {
if (!file.getParentFile().isDirectory()) {
throw new IOException("Mkdirs failed to create " +
- file.getParentFile().toString());
+ file.getParentFile().toString());
}
}
OutputStream out = new FileOutputStream(file);
try {
- byte[] buffer = new byte[BUFFER_SIZE];
+ byte[] buffer = new byte[8192];
int i;
while ((i = in.read(buffer)) != -1) {
out.write(buffer, 0, i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 7e76099..3c31a8c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -814,7 +814,7 @@ public class RPC {
static String serverNameFromClass(Class<?> clazz) {
//The basic idea here is to handle names like
- //org.apache.hadoop.ozone.protocol.proto.
+ //org.apache.hadoop.hdsl.protocol.proto.
//
// StorageDatanodeProtocolProtos$StorageContainerDatanodeProtocolService$2
//where the getSimpleName is also empty
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 7b885c9..4f68ace 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -68,6 +68,44 @@
<artifactId>hadoop-client-integration-tests</artifactId>
<scope>provided</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-ozone-manager</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-server-scm</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cblock-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-container-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-objectstore-service</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-tools</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cblock-tools</artifactId>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml b/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
index fefb37f..b88514e 100644
--- a/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/cblock/docker-compose.yaml
@@ -17,7 +17,7 @@
version: "3"
services:
namenode:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
hostname: namenode
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
@@ -29,36 +29,38 @@ services:
- ./docker-config
command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
ports:
- 9864
- command: ["/opt/hadoop/bin/hdfs","datanode"]
+ command: ["/opt/hadoop/bin/oz","datanode"]
+ env_file:
+ - ./docker-config
jscsi:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
ports:
- 3260:3260
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
env_file:
- ./docker-config
- command: ["/opt/hadoop/bin/hdfs","jscsi"]
+ command: ["/opt/hadoop/bin/oz","jscsi"]
cblock:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
env_file:
- ./docker-config
- command: ["/opt/hadoop/bin/hdfs","cblockserver"]
+ command: ["/opt/hadoop/bin/oz","cblockserver"]
scm:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
ports:
- 9876:9876
env_file:
- ./docker-config
- command: ["/opt/hadoop/bin/hdfs","scm"]
+ command: ["/opt/hadoop/bin/oz","scm"]
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/cblock/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/cblock/docker-config b/hadoop-dist/src/main/compose/cblock/docker-config
index 6851bc6..95917fd 100644
--- a/hadoop-dist/src/main/compose/cblock/docker-config
+++ b/hadoop-dist/src/main/compose/cblock/docker-config
@@ -27,7 +27,7 @@ OZONE-SITE.XML_ozone.scm.client.address=scm
OZONE-SITE.XML_dfs.cblock.jscsi.cblock.server.address=cblock
OZONE-SITE.XML_dfs.cblock.scm.ipaddress=scm
OZONE-SITE.XML_dfs.cblock.service.leveldb.path=/tmp
-
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HdslServerPlugin,org.apache.hadoop.ozone.web.ObjectStoreRestPlugin
HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
index 02713c7..f2b263c 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
+++ b/hadoop-dist/src/main/compose/ozone/docker-compose.yaml
@@ -17,7 +17,7 @@
version: "3"
services:
namenode:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
hostname: namenode
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
@@ -29,14 +29,16 @@ services:
- ./docker-config
command: ["/opt/hadoop/bin/hdfs","namenode"]
datanode:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
ports:
- 9864
- command: ["/opt/hadoop/bin/hdfs","datanode"]
+ command: ["/opt/hadoop/bin/oz","datanode"]
+ env_file:
+ - ./docker-config
ksm:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
ports:
@@ -45,9 +47,9 @@ services:
ENSURE_KSM_INITIALIZED: /data/metadata/ksm/current/VERSION
env_file:
- ./docker-config
- command: ["/opt/hadoop/bin/hdfs","ksm"]
+ command: ["/opt/hadoop/bin/oz","ksm"]
scm:
- image: elek/hadoop-runner:latest
+ image: elek/hadoop-runner:o3-refactor
volumes:
- ../..//hadoop-${VERSION}:/opt/hadoop
ports:
@@ -56,4 +58,4 @@ services:
- ./docker-config
environment:
ENSURE_SCM_INITIALIZED: /data/metadata/scm/current/VERSION
- command: ["/opt/hadoop/bin/hdfs","scm"]
+ command: ["/opt/hadoop/bin/oz","scm"]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-dist/src/main/compose/ozone/docker-config
----------------------------------------------------------------------
diff --git a/hadoop-dist/src/main/compose/ozone/docker-config b/hadoop-dist/src/main/compose/ozone/docker-config
index c3f3663..20c1e30 100644
--- a/hadoop-dist/src/main/compose/ozone/docker-config
+++ b/hadoop-dist/src/main/compose/ozone/docker-config
@@ -27,6 +27,7 @@ HDFS-SITE.XML_dfs.namenode.rpc-address=namenode:9000
HDFS-SITE.XML_dfs.namenode.name.dir=/data/namenode
HDFS-SITE.XML_rpc.metrics.quantile.enable=true
HDFS-SITE.XML_rpc.metrics.percentiles.intervals=60,300
+HDFS-SITE.XML_dfs.datanode.plugins=org.apache.hadoop.ozone.HdslServerPlugin,org.apache.hadoop.ozone.web.ObjectStoreRestPlugin
LOG4J.PROPERTIES_log4j.rootLogger=INFO, stdout
LOG4J.PROPERTIES_log4j.appender.stdout=org.apache.log4j.ConsoleAppender
LOG4J.PROPERTIES_log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
index 5503cb2..8e2bc94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/dev-support/findbugsExcludeFile.xml
@@ -63,13 +63,6 @@
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
- <Match>
- <Package name="org.apache.hadoop.ozone.protocol.proto" />
- </Match>
- <Match>
- <Package name="org.apache.hadoop.hdfs.ozone.protocol.proto" />
- </Match>
-
<!-- BlockLocations are user-facing, but LocatedBlocks are not. -->
<Match>
<Class name="org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus" />
@@ -98,4 +91,5 @@
<Method name="getSymlinkInBytes" />
<Bug pattern="EI_EXPOSE_REP" />
</Match>
+
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index 19d8113..a5ed7a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -64,6 +64,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.mock-server</groupId>
<artifactId>mockserver-netty</artifactId>
<scope>test</scope>
@@ -100,37 +105,14 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
<type>test-jar</type>
</dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-annotations</artifactId>
- </dependency>
- <dependency>
- <groupId>io.netty</groupId>
- <artifactId>netty-all</artifactId>
- </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
-
- <dependency>
- <artifactId>ratis-server</artifactId>
- <groupId>org.apache.ratis</groupId>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <artifactId>ratis-netty</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
- <dependency>
- <artifactId>ratis-grpc</artifactId>
- <groupId>org.apache.ratis</groupId>
- </dependency>
</dependencies>
<build>
@@ -177,11 +159,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>inotify.proto</include>
<include>erasurecoding.proto</include>
<include>ReconfigurationProtocol.proto</include>
- <include>StorageContainerLocationProtocol.proto</include>
- <include>DatanodeContainerProtocol.proto</include>
- <include>Ozone.proto</include>
- <include>KeySpaceManagerProtocol.proto</include>
- <include>ScmBlockLocationProtocol.proto</include>
</includes>
</source>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
deleted file mode 100644
index fde728c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import static java.lang.Thread.NORM_PRIORITY;
-
-/**
- * This class contains constants for configuration keys used in CBlock.
- */
-public final class CBlockConfigKeys {
- public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
- "dfs.cblock.servicerpc-address";
- public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
- 9810;
- public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
- "0.0.0.0";
-
- public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
- "dfs.cblock.jscsi-address";
-
- //The port on CBlockManager node for jSCSI to ask
- public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
- "dfs.cblock.jscsi.port";
- public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
- 9811;
-
- public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
- "dfs.cblock.service.rpc-bind-host";
- public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY =
- "dfs.cblock.jscsi.rpc-bind-host";
-
- // default block size is 4KB
- public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT =
- 4096;
-
- public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY =
- "dfs.cblock.service.handler.count";
- public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10;
-
- public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY =
- "dfs.cblock.service.leveldb.path";
- //TODO : find a better place
- public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT =
- "/tmp/cblock_levelDB.dat";
-
-
- public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY =
- "dfs.cblock.disk.cache.path";
- public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT =
- "/tmp/cblockCacheDB";
- /**
- * Setting this flag to true makes the block layer compute a sha256 hash of
- * the data and log that information along with block ID. This is very
- * useful for doing trace based simulation of various workloads. Since it is
- * computing a hash for each block this could be expensive, hence default
- * is false.
- */
- public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io";
- public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false;
-
- public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO =
- "dfs.cblock.short.circuit.io";
- public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT =
- false;
-
- /**
- * Cache size in 1000s of entries. 256 indicates 256 * 1024.
- */
- public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB =
- "dfs.cblock.cache.queue.size.in.kb";
- public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256;
-
- /**
- * Minimum Number of threads that cache pool will use for background I/O.
- */
- public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
- "dfs.cblock.cache.core.min.pool.size";
- public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
-
- /**
- * Maximum Number of threads that cache pool will use for background I/O.
- */
-
- public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE =
- "dfs.cblock.cache.max.pool.size";
- public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256;
-
- /**
- * Number of seconds to keep the Thread alive when it is idle.
- */
- public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE =
- "dfs.cblock.cache.keep.alive";
- public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT = "60s";
-
- /**
- * Priority of cache flusher thread, affecting the relative performance of
- * write and read.
- */
- public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY =
- "dfs.cblock.cache.thread.priority";
- public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT =
- NORM_PRIORITY;
-
- /**
- * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs.
- */
- public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE =
- "dfs.cblock.cache.block.buffer.size";
- public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512;
-
- public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL =
- "dfs.cblock.block.buffer.flush.interval";
- public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT =
- "60s";
-
- // jscsi server settings
- public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY =
- "dfs.cblock.jscsi.server.address";
- public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT =
- "0.0.0.0";
- public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY =
- "dfs.cblock.jscsi.cblock.server.address";
- public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
- "127.0.0.1";
-
- // to what address cblock server should talk to scm?
- public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
- "dfs.cblock.scm.ipaddress";
- public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
- "127.0.0.1";
- public static final String DFS_CBLOCK_SCM_PORT_KEY =
- "dfs.cblock.scm.port";
- public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
-
- public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
- "dfs.cblock.container.size.gb";
- public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
- 5;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB.
- public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY =
- "dfs.cblock.cache.leveldb.cache.size.mb";
- public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256;
-
- /**
- * Cache does an best case attempt to write a block to a container.
- * At some point of time, we will need to handle the case where we did try
- * 64K times and is till not able to write to the container.
- *
- * TODO: We will need cBlock Server to allow us to do a remapping of the
- * block location in case of failures, at that point we should reduce the
- * retry count to a more normal number. This is approximately 18 hours of
- * retry.
- */
- public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY =
- "dfs.cblock.cache.max.retry";
- public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT =
- 64 * 1024;
-
- /**
- * Cblock CLI configs.
- */
- public static final String DFS_CBLOCK_MANAGER_POOL_SIZE =
- "dfs.cblock.manager.pool.size";
- public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16;
-
- /**
- * currently the largest supported volume is about 8TB, which might take
- * > 20 seconds to finish creating containers. thus set timeout to 30 sec.
- */
- public static final String DFS_CBLOCK_RPC_TIMEOUT =
- "dfs.cblock.rpc.timeout";
- public static final String DFS_CBLOCK_RPC_TIMEOUT_DEFAULT = "300s";
-
- public static final String DFS_CBLOCK_ISCSI_ADVERTISED_IP =
- "dfs.cblock.iscsi.advertised.ip";
-
- public static final String DFS_CBLOCK_ISCSI_ADVERTISED_PORT =
- "dfs.cblock.iscsi.advertised.port";
-
- public static final int DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT = 3260;
-
-
- public static final String
- DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED
- = "dfs.cblock.kubernetes.dynamic-provisioner.enabled";
-
- public static final boolean
- DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT = false;
-
- public static final String
- DFS_CBLOCK_KUBERNETES_CBLOCK_USER =
- "dfs.cblock.kubernetes.cblock-user";
-
- public static final String
- DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT =
- "iqn.2001-04.org.apache.hadoop";
-
- public static final String
- DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY =
- "dfs.cblock.kubernetes.configfile";
-
- private CBlockConfigKeys() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
deleted file mode 100644
index 5f5d3cd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/cblock/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-/**
- This package contains ozone client side libraries.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
index 517e474..96f22ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeID.java
@@ -7,7 +7,7 @@
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -54,6 +54,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
private String xferAddr;
private int containerPort; // container Stand_alone Rpc port.
private int ratisPort; // Container Ratis RPC Port.
+ private int ozoneRestPort;
/**
* UUID identifying a given datanode. For upgraded Datanodes this is the
@@ -75,6 +76,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
from.getInfoPort(),
from.getInfoSecurePort(),
from.getIpcPort());
+ this.ozoneRestPort = from.getOzoneRestPort();
this.peerHostName = from.getPeerHostName();
}
@@ -267,6 +269,8 @@ public class DatanodeID implements Comparable<DatanodeID> {
infoPort = nodeReg.getInfoPort();
infoSecurePort = nodeReg.getInfoSecurePort();
ipcPort = nodeReg.getIpcPort();
+ ratisPort = nodeReg.getRatisPort();
+ ozoneRestPort = nodeReg.getOzoneRestPort();
}
/**
@@ -313,6 +317,24 @@ public class DatanodeID implements Comparable<DatanodeID> {
}
/**
+ * Ozone rest port.
+ *
+ * @return rest port.
+ */
+ public int getOzoneRestPort() {
+ return ozoneRestPort;
+ }
+
+ /**
+ * Set the ozone rest port.
+ *
+ * @param ozoneRestPort
+ */
+ public void setOzoneRestPort(int ozoneRestPort) {
+ this.ozoneRestPort = ozoneRestPort;
+ }
+
+ /**
* Returns a DataNode ID from the protocol buffers.
*
* @param datanodeIDProto - protoBuf Message
@@ -326,6 +348,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
datanodeIDProto.getInfoSecurePort(), datanodeIDProto.getIpcPort());
id.setContainerPort(datanodeIDProto.getContainerPort());
id.setRatisPort(datanodeIDProto.getRatisPort());
+ id.setOzoneRestPort(datanodeIDProto.getOzoneRestPort());
return id;
}
@@ -345,6 +368,7 @@ public class DatanodeID implements Comparable<DatanodeID> {
.setIpcPort(this.getIpcPort())
.setContainerPort(this.getContainerPort())
.setRatisPort(this.getRatisPort())
+ .setOzoneRestPort(this.getOzoneRestPort())
.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index d9e7aa0..e94b094 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -292,7 +292,9 @@ public class PBHelperClient {
dn.getDatanodeUuid() : "")
.setInfoPort(dn.getInfoPort())
.setInfoSecurePort(dn.getInfoSecurePort())
- .setIpcPort(dn.getIpcPort()).build();
+ .setIpcPort(dn.getIpcPort())
+ .setOzoneRestPort(dn.getOzoneRestPort())
+ .build();
}
public static DatanodeInfoProto.AdminState convert(
@@ -742,9 +744,13 @@ public class PBHelperClient {
// DatanodeId
public static DatanodeID convert(DatanodeIDProto dn) {
- return new DatanodeID(dn.getIpAddr(), dn.getHostName(),
- dn.getDatanodeUuid(), dn.getXferPort(), dn.getInfoPort(),
- dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0, dn.getIpcPort());
+ DatanodeID datanodeID =
+ new DatanodeID(dn.getIpAddr(), dn.getHostName(), dn.getDatanodeUuid(),
+ dn.getXferPort(), dn.getInfoPort(),
+ dn.hasInfoSecurePort() ? dn.getInfoSecurePort() : 0,
+ dn.getIpcPort());
+ datanodeID.setOzoneRestPort(dn.getOzoneRestPort());
+ return datanodeID;
}
public static AdminStates convert(AdminState adminState) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
deleted file mode 100644
index ff0ac4e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneAcl.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.ozone;
-
-import java.util.Objects;
-
-/**
- * OzoneACL classes define bucket ACLs used in OZONE.
- *
- * ACLs in Ozone follow this pattern.
- * • user:name:rw
- * • group:name:rw
- * • world::rw
- */
-public class OzoneAcl {
- private OzoneACLType type;
- private String name;
- private OzoneACLRights rights;
-
- /**
- * Constructor for OzoneAcl.
- */
- public OzoneAcl() {
- }
-
- /**
- * Constructor for OzoneAcl.
- *
- * @param type - Type
- * @param name - Name of user
- * @param rights - Rights
- */
- public OzoneAcl(OzoneACLType type, String name, OzoneACLRights rights) {
- this.name = name;
- this.rights = rights;
- this.type = type;
- if (type == OzoneACLType.WORLD && name.length() != 0) {
- throw new IllegalArgumentException("Unexpected name part in world type");
- }
- if (((type == OzoneACLType.USER) || (type == OzoneACLType.GROUP))
- && (name.length() == 0)) {
- throw new IllegalArgumentException("User or group name is required");
- }
- }
-
- /**
- * Parses an ACL string and returns the ACL object.
- *
- * @param acl - Acl String , Ex. user:anu:rw
- *
- * @return - Ozone ACLs
- */
- public static OzoneAcl parseAcl(String acl) throws IllegalArgumentException {
- if ((acl == null) || acl.isEmpty()) {
- throw new IllegalArgumentException("ACLs cannot be null or empty");
- }
- String[] parts = acl.trim().split(":");
- if (parts.length < 3) {
- throw new IllegalArgumentException("ACLs are not in expected format");
- }
-
- OzoneACLType aclType = OzoneACLType.valueOf(parts[0].toUpperCase());
- OzoneACLRights rights = OzoneACLRights.getACLRight(parts[2].toLowerCase());
-
- // TODO : Support sanitation of these user names by calling into
- // userAuth Interface.
- return new OzoneAcl(aclType, parts[1], rights);
- }
-
- @Override
- public String toString() {
- return type + ":" + name + ":" + OzoneACLRights.getACLRightsString(rights);
- }
-
- /**
- * Returns a hash code value for the object. This method is
- * supported for the benefit of hash tables.
- *
- * @return a hash code value for this object.
- *
- * @see Object#equals(Object)
- * @see System#identityHashCode
- */
- @Override
- public int hashCode() {
- return Objects.hash(this.getName(), this.getRights().toString(),
- this.getType().toString());
- }
-
- /**
- * Returns name.
- *
- * @return name
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns Rights.
- *
- * @return - Rights
- */
- public OzoneACLRights getRights() {
- return rights;
- }
-
- /**
- * Returns Type.
- *
- * @return type
- */
- public OzoneACLType getType() {
- return type;
- }
-
- /**
- * Indicates whether some other object is "equal to" this one.
- *
- * @param obj the reference object with which to compare.
- *
- * @return {@code true} if this object is the same as the obj
- * argument; {@code false} otherwise.
- */
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- OzoneAcl otherAcl = (OzoneAcl) obj;
- return otherAcl.getName().equals(this.getName()) &&
- otherAcl.getRights() == this.getRights() &&
- otherAcl.getType() == this.getType();
- }
-
- /**
- * ACL types.
- */
- public enum OzoneACLType {
- USER(OzoneConsts.OZONE_ACL_USER_TYPE),
- GROUP(OzoneConsts.OZONE_ACL_GROUP_TYPE),
- WORLD(OzoneConsts.OZONE_ACL_WORLD_TYPE);
-
- /**
- * String value for this Enum.
- */
- private final String value;
-
- /**
- * Init OzoneACLtypes enum.
- *
- * @param val String type for this enum.
- */
- OzoneACLType(String val) {
- value = val;
- }
- }
-
- /**
- * ACL rights.
- */
- public enum OzoneACLRights {
- READ, WRITE, READ_WRITE;
-
- /**
- * Returns the ACL rights based on passed in String.
- *
- * @param type ACL right string
- *
- * @return OzoneACLRights
- */
- public static OzoneACLRights getACLRight(String type) {
- if (type == null || type.isEmpty()) {
- throw new IllegalArgumentException("ACL right cannot be empty");
- }
-
- switch (type) {
- case OzoneConsts.OZONE_ACL_READ:
- return OzoneACLRights.READ;
- case OzoneConsts.OZONE_ACL_WRITE:
- return OzoneACLRights.WRITE;
- case OzoneConsts.OZONE_ACL_READ_WRITE:
- case OzoneConsts.OZONE_ACL_WRITE_READ:
- return OzoneACLRights.READ_WRITE;
- default:
- throw new IllegalArgumentException("ACL right is not recognized");
- }
-
- }
-
- /**
- * Returns String representation of ACL rights.
- * @param acl OzoneACLRights
- * @return String representation of acl
- */
- public static String getACLRightsString(OzoneACLRights acl) {
- switch(acl) {
- case READ:
- return OzoneConsts.OZONE_ACL_READ;
- case WRITE:
- return OzoneConsts.OZONE_ACL_WRITE;
- case READ_WRITE:
- return OzoneConsts.OZONE_ACL_READ_WRITE;
- default:
- throw new IllegalArgumentException("ACL right is not recognized");
- }
- }
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
deleted file mode 100644
index b3b35f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.client.rest.RestClient;
-import org.apache.hadoop.ozone.client.rpc.RpcClient;
-import org.apache.hadoop.scm.ScmConfigKeys;
-
-/**
- * This class contains constants for configuration keys used in Ozone.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class OzoneConfigKeys {
- public static final String DFS_CONTAINER_IPC_PORT =
- "dfs.container.ipc";
- public static final int DFS_CONTAINER_IPC_PORT_DEFAULT = 9859;
-
- /**
- *
- * When set to true, allocate a random free port for ozone container,
- * so that a mini cluster is able to launch multiple containers on a node.
- *
- * When set to false (default), container port is fixed as specified by
- * DFS_CONTAINER_IPC_PORT_DEFAULT.
- */
- public static final String DFS_CONTAINER_IPC_RANDOM_PORT =
- "dfs.container.ipc.random.port";
- public static final boolean DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT =
- false;
-
- /**
- * Ratis Port where containers listen to.
- */
- public static final String DFS_CONTAINER_RATIS_IPC_PORT =
- "dfs.container.ratis.ipc";
- public static final int DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT = 9858;
-
- /**
- * When set to true, allocate a random free port for ozone container, so that
- * a mini cluster is able to launch multiple containers on a node.
- */
- public static final String DFS_CONTAINER_RATIS_IPC_RANDOM_PORT =
- "dfs.container.ratis.ipc.random.port";
- public static final boolean DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT =
- false;
-
- public static final String OZONE_LOCALSTORAGE_ROOT =
- "ozone.localstorage.root";
- public static final String OZONE_LOCALSTORAGE_ROOT_DEFAULT = "/tmp/ozone";
- public static final String OZONE_ENABLED =
- "ozone.enabled";
- public static final boolean OZONE_ENABLED_DEFAULT = false;
- public static final String OZONE_HANDLER_TYPE_KEY =
- "ozone.handler.type";
- public static final String OZONE_HANDLER_TYPE_DEFAULT = "distributed";
- public static final String OZONE_TRACE_ENABLED_KEY =
- "ozone.trace.enabled";
- public static final boolean OZONE_TRACE_ENABLED_DEFAULT = false;
-
- public static final String OZONE_METADATA_DIRS =
- "ozone.metadata.dirs";
-
- public static final String OZONE_METADATA_STORE_IMPL =
- "ozone.metastore.impl";
- public static final String OZONE_METADATA_STORE_IMPL_LEVELDB =
- "LevelDB";
- public static final String OZONE_METADATA_STORE_IMPL_ROCKSDB =
- "RocksDB";
- public static final String OZONE_METADATA_STORE_IMPL_DEFAULT =
- OZONE_METADATA_STORE_IMPL_ROCKSDB;
-
- public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS =
- "ozone.metastore.rocksdb.statistics";
-
- public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT =
- "ALL";
- public static final String OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF =
- "OFF";
-
- public static final String OZONE_CONTAINER_CACHE_SIZE =
- "ozone.container.cache.size";
- public static final int OZONE_CONTAINER_CACHE_DEFAULT = 1024;
-
- public static final String OZONE_SCM_BLOCK_SIZE_IN_MB =
- "ozone.scm.block.size.in.mb";
- public static final long OZONE_SCM_BLOCK_SIZE_DEFAULT = 256;
-
- /**
- * Ozone administrator users delimited by comma.
- * If not set, only the user who launches an ozone service will be the
- * admin user. This property must be set if ozone services are started by
- * different users. Otherwise the RPC layer will reject calls from
- * other servers which are started by users not in the list.
- * */
- public static final String OZONE_ADMINISTRATORS =
- "ozone.administrators";
-
- public static final String OZONE_CLIENT_PROTOCOL =
- "ozone.client.protocol";
- public static final Class<? extends ClientProtocol>
- OZONE_CLIENT_PROTOCOL_RPC = RpcClient.class;
- public static final Class<? extends ClientProtocol>
- OZONE_CLIENT_PROTOCOL_REST = RestClient.class;
-
- // This defines the overall connection limit for the connection pool used in
- // RestClient.
- public static final String OZONE_REST_CLIENT_HTTP_CONNECTION_MAX =
- "ozone.rest.client.http.connection.max";
- public static final int OZONE_REST_CLIENT_HTTP_CONNECTION_DEFAULT = 100;
-
- // This defines the connection limit per one HTTP route/host.
- public static final String OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX =
- "ozone.rest.client.http.connection.per-route.max";
-
- public static final int
- OZONE_REST_CLIENT_HTTP_CONNECTION_PER_ROUTE_MAX_DEFAULT = 20;
-
- public static final String OZONE_CLIENT_SOCKET_TIMEOUT =
- "ozone.client.socket.timeout";
- public static final int OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT = 5000;
- public static final String OZONE_CLIENT_CONNECTION_TIMEOUT =
- "ozone.client.connection.timeout";
- public static final int OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT = 5000;
-
- public static final String OZONE_REPLICATION = "ozone.replication";
- public static final int OZONE_REPLICATION_DEFAULT =
- ReplicationFactor.THREE.getValue();
-
- public static final String OZONE_REPLICATION_TYPE = "ozone.replication.type";
- public static final String OZONE_REPLICATION_TYPE_DEFAULT =
- ReplicationType.RATIS.toString();
-
- /**
- * Configuration property to configure the cache size of client list calls.
- */
- public static final String OZONE_CLIENT_LIST_CACHE_SIZE =
- "ozone.client.list.cache";
- public static final int OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT = 1000;
-
- /**
- * Configuration properties for Ozone Block Deleting Service.
- */
- public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL =
- "ozone.block.deleting.service.interval";
- public static final String OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT
- = "60s";
-
- /**
- * The interval of open key clean service.
- */
- public static final String OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS =
- "ozone.open.key.cleanup.service.interval.seconds";
- public static final int
- OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT
- = 24 * 3600; // a total of 24 hour
-
- /**
- * An open key gets cleaned up when it is being in open state for too long.
- */
- public static final String OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS =
- "ozone.open.key.expire.threshold";
- public static final int OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT =
- 24 * 3600;
-
- public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT =
- "ozone.block.deleting.service.timeout";
- public static final String OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT
- = "300s"; // 300s for default
-
- public static final String OZONE_KEY_PREALLOCATION_MAXSIZE =
- "ozone.key.preallocation.maxsize";
- public static final long OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT
- = 128 * OzoneConsts.MB;
-
- public static final String OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER =
- "ozone.block.deleting.limit.per.task";
- public static final int OZONE_BLOCK_DELETING_LIMIT_PER_CONTAINER_DEFAULT
- = 1000;
-
- public static final String OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL
- = "ozone.block.deleting.container.limit.per.interval";
- public static final int
- OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT = 10;
-
- public static final String OZONE_CONTAINER_REPORT_INTERVAL =
- "ozone.container.report.interval";
- public static final String OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT =
- "60s";
-
- public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
- public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
- public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY;
- public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT;
- public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY;
- public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY;
- public static final int DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT
- = ScmConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT;
- public static final int DFS_CONTAINER_CHUNK_MAX_SIZE
- = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
- public static final String DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR =
- "dfs.container.ratis.datanode.storage.dir";
-
- public static final String OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL =
- "ozone.web.authentication.kerberos.principal";
-
- /**
- * There is no need to instantiate this class.
- */
- private OzoneConfigKeys() {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
deleted file mode 100644
index 3cc4697..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Set of constants used in Ozone implementation.
- */
-@InterfaceAudience.Private
-public final class OzoneConsts {
- public static final String OZONE_SIMPLE_ROOT_USER = "root";
- public static final String OZONE_SIMPLE_HDFS_USER = "hdfs";
-
- /*
- * BucketName length is used for both buckets and volume lengths
- */
- public static final int OZONE_MIN_BUCKET_NAME_LENGTH = 3;
- public static final int OZONE_MAX_BUCKET_NAME_LENGTH = 63;
-
- public static final String OZONE_ACL_USER_TYPE = "user";
- public static final String OZONE_ACL_GROUP_TYPE = "group";
- public static final String OZONE_ACL_WORLD_TYPE = "world";
-
- public static final String OZONE_ACL_READ = "r";
- public static final String OZONE_ACL_WRITE = "w";
- public static final String OZONE_ACL_READ_WRITE = "rw";
- public static final String OZONE_ACL_WRITE_READ = "wr";
-
- public static final String OZONE_DATE_FORMAT =
- "EEE, dd MMM yyyy HH:mm:ss zzz";
- public static final String OZONE_TIME_ZONE = "GMT";
-
- public static final String OZONE_COMPONENT = "component";
- public static final String OZONE_FUNCTION = "function";
- public static final String OZONE_RESOURCE = "resource";
- public static final String OZONE_USER = "user";
- public static final String OZONE_REQUEST = "request";
-
- public static final String CONTAINER_EXTENSION = ".container";
- public static final String CONTAINER_META = ".meta";
-
- // container storage is in the following format.
- // Data Volume basePath/containers/<containerName>/metadata and
- // Data Volume basePath/containers/<containerName>/data/...
- public static final String CONTAINER_PREFIX = "containers";
- public static final String CONTAINER_META_PATH = "metadata";
- public static final String CONTAINER_DATA_PATH = "data";
- public static final String CONTAINER_TEMPORARY_CHUNK_PREFIX = "tmp";
- public static final String CONTAINER_CHUNK_NAME_DELIMITER = ".";
- public static final String CONTAINER_ROOT_PREFIX = "repository";
-
- public static final String FILE_HASH = "SHA-256";
- public final static String CHUNK_OVERWRITE = "OverWriteRequested";
-
- public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1 MB
- public static final long KB = 1024L;
- public static final long MB = KB * 1024L;
- public static final long GB = MB * 1024L;
- public static final long TB = GB * 1024L;
-
- /**
- * level DB names used by SCM and data nodes.
- */
- public static final String CONTAINER_DB_SUFFIX = "container.db";
- public static final String SCM_CONTAINER_DB = "scm-" + CONTAINER_DB_SUFFIX;
- public static final String DN_CONTAINER_DB = "-dn-"+ CONTAINER_DB_SUFFIX;
- public static final String BLOCK_DB = "block.db";
- public static final String NODEPOOL_DB = "nodepool.db";
- public static final String OPEN_CONTAINERS_DB = "openContainers.db";
- public static final String DELETED_BLOCK_DB = "deletedBlock.db";
- public static final String KSM_DB_NAME = "ksm.db";
-
- /**
- * Supports Bucket Versioning.
- */
- public enum Versioning {NOT_DEFINED, ENABLED, DISABLED}
-
- /**
- * Ozone handler types.
- */
- public static final String OZONE_HANDLER_DISTRIBUTED = "distributed";
- public static final String OZONE_HANDLER_LOCAL = "local";
-
- public static final String DELETING_KEY_PREFIX = "#deleting#";
- public static final String OPEN_KEY_PREFIX = "#open#";
- public static final String OPEN_KEY_ID_DELIMINATOR = "#";
-
- /**
- * KSM LevelDB prefixes.
- *
- * KSM DB stores metadata as KV pairs with certain prefixes,
- * prefix is used to improve the performance to get related
- * metadata.
- *
- * KSM DB Schema:
- * ----------------------------------------------------------
- * | KEY | VALUE |
- * ----------------------------------------------------------
- * | $userName | VolumeList |
- * ----------------------------------------------------------
- * | /#volumeName | VolumeInfo |
- * ----------------------------------------------------------
- * | /#volumeName/#bucketName | BucketInfo |
- * ----------------------------------------------------------
- * | /volumeName/bucketName/keyName | KeyInfo |
- * ----------------------------------------------------------
- * | #deleting#/volumeName/bucketName/keyName | KeyInfo |
- * ----------------------------------------------------------
- */
- public static final String KSM_VOLUME_PREFIX = "/#";
- public static final String KSM_BUCKET_PREFIX = "/#";
- public static final String KSM_KEY_PREFIX = "/";
- public static final String KSM_USER_PREFIX = "$";
-
- /**
- * Max KSM Quota size of 1024 PB.
- */
- public static final long MAX_QUOTA_IN_BYTES = 1024L * 1024 * TB;
-
- /**
- * Max number of keys returned per list buckets operation.
- */
- public static final int MAX_LISTBUCKETS_SIZE = 1024;
-
- /**
- * Max number of keys returned per list keys operation.
- */
- public static final int MAX_LISTKEYS_SIZE = 1024;
-
- /**
- * Max number of volumes returned per list volumes operation.
- */
- public static final int MAX_LISTVOLUMES_SIZE = 1024;
-
- public static final int INVALID_PORT = -1;
-
-
- // The ServiceListJSONServlet context attribute where KeySpaceManager
- // instance gets stored.
- public static final String KSM_CONTEXT_ATTRIBUTE = "ozone.ksm";
-
- private OzoneConsts() {
- // Never Constructed
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
deleted file mode 100644
index 39b7bb8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.util.List;
-
-/**
- * This class encapsulates the arguments that are
- * required for creating a bucket.
- */
-public final class BucketArgs {
-
- /**
- * ACL Information.
- */
- private List<OzoneAcl> acls;
- /**
- * Bucket Version flag.
- */
- private Boolean versioning;
- /**
- * Type of storage to be used for this bucket.
- * [RAM_DISK, SSD, DISK, ARCHIVE]
- */
- private StorageType storageType;
-
- /**
- * Private constructor, constructed via builder.
- * @param versioning Bucket version flag.
- * @param storageType Storage type to be used.
- * @param acls list of ACLs.
- */
- private BucketArgs(Boolean versioning, StorageType storageType,
- List<OzoneAcl> acls) {
- this.acls = acls;
- this.versioning = versioning;
- this.storageType = storageType;
- }
-
- /**
- * Returns true if bucket version is enabled, else false.
- * @return isVersionEnabled
- */
- public Boolean getVersioning() {
- return versioning;
- }
-
- /**
- * Returns the type of storage to be used.
- * @return StorageType
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Returns the ACL's associated with this bucket.
- * @return List<OzoneAcl>
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Returns new builder class that builds a KsmBucketInfo.
- *
- * @return Builder
- */
- public static BucketArgs.Builder newBuilder() {
- return new BucketArgs.Builder();
- }
-
- /**
- * Builder for KsmBucketInfo.
- */
- public static class Builder {
- private Boolean versioning;
- private StorageType storageType;
- private List<OzoneAcl> acls;
-
- public BucketArgs.Builder setVersioning(Boolean versionFlag) {
- this.versioning = versionFlag;
- return this;
- }
-
- public BucketArgs.Builder setStorageType(StorageType storage) {
- this.storageType = storage;
- return this;
- }
-
- public BucketArgs.Builder setAcls(List<OzoneAcl> listOfAcls) {
- this.acls = listOfAcls;
- return this;
- }
-
- /**
- * Constructs the BucketArgs.
- * @return instance of BucketArgs.
- */
- public BucketArgs build() {
- return new BucketArgs(versioning, storageType, acls);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
deleted file mode 100644
index 1ed41f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ObjectStore.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * ObjectStore class is responsible for the client operations that can be
- * performed on Ozone Object Store.
- */
-public class ObjectStore {
-
- /**
- * The proxy used for connecting to the cluster and perform
- * client operations.
- */
- private final ClientProtocol proxy;
-
- /**
- * Cache size to be used for listVolume calls.
- */
- private int listCacheSize;
-
- /**
- * Creates an instance of ObjectStore.
- * @param conf Configuration object.
- * @param proxy ClientProtocol proxy.
- */
- public ObjectStore(Configuration conf, ClientProtocol proxy) {
- this.proxy = proxy;
- this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
- }
-
- /**
- * Creates the volume with default values.
- * @param volumeName Name of the volume to be created.
- * @throws IOException
- */
- public void createVolume(String volumeName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- OzoneClientUtils.verifyResourceName(volumeName);
- proxy.createVolume(volumeName);
- }
-
- /**
- * Creates the volume.
- * @param volumeName Name of the volume to be created.
- * @param volumeArgs Volume properties.
- * @throws IOException
- */
- public void createVolume(String volumeName, VolumeArgs volumeArgs)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(volumeArgs);
- OzoneClientUtils.verifyResourceName(volumeName);
- proxy.createVolume(volumeName, volumeArgs);
- }
-
- /**
- * Returns the volume information.
- * @param volumeName Name of the volume.
- * @return OzoneVolume
- * @throws IOException
- */
- public OzoneVolume getVolume(String volumeName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- OzoneClientUtils.verifyResourceName(volumeName);
- OzoneVolume volume = proxy.getVolumeDetails(volumeName);
- return volume;
- }
-
-
- /**
- * Returns Iterator to iterate over all the volumes in object store.
- * The result can be restricted using volume prefix, will return all
- * volumes if volume prefix is null.
- *
- * @param volumePrefix Volume prefix to match
- * @return {@code Iterator<OzoneVolume>}
- */
- public Iterator<OzoneVolume> listVolumes(String volumePrefix)
- throws IOException {
- return new VolumeIterator(volumePrefix);
- }
-
- /**
- * Returns Iterator to iterate over the List of volumes owned by a specific
- * user. The result can be restricted using volume prefix, will return all
- * volumes if volume prefix is null. If user is null, returns the volume of
- * current user.
- *
- * @param user User Name
- * @param volumePrefix Volume prefix to match
- * @return {@code Iterator<OzoneVolume>}
- */
- public Iterator<OzoneVolume> listVolumes(String user, String volumePrefix)
- throws IOException {
- if(Strings.isNullOrEmpty(user)) {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
- }
- return new VolumeIterator(user, volumePrefix);
- }
-
- /**
- * Deletes the volume.
- * @param volumeName Name of the volume.
- * @throws IOException
- */
- public void deleteVolume(String volumeName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- OzoneClientUtils.verifyResourceName(volumeName);
- proxy.deleteVolume(volumeName);
- }
-
- /**
- * An Iterator to iterate over {@link OzoneVolume} list.
- */
- private class VolumeIterator implements Iterator<OzoneVolume> {
-
- private String user = null;
- private String volPrefix = null;
-
- private Iterator<OzoneVolume> currentIterator;
- private OzoneVolume currentValue;
-
- /**
- * Creates an Iterator to iterate over all volumes in the cluster,
- * which matches the volume prefix.
- * @param volPrefix prefix to match
- */
- VolumeIterator(String volPrefix) {
- this(null, volPrefix);
- }
-
- /**
- * Creates an Iterator to iterate over all volumes of the user,
- * which matches volume prefix.
- * @param user user name
- * @param volPrefix volume prefix to match
- */
- VolumeIterator(String user, String volPrefix) {
- this.user = user;
- this.volPrefix = volPrefix;
- this.currentValue = null;
- this.currentIterator = getNextListOfVolumes(null).iterator();
- }
-
- @Override
- public boolean hasNext() {
- if(!currentIterator.hasNext()) {
- currentIterator = getNextListOfVolumes(
- currentValue != null ? currentValue.getName() : null)
- .iterator();
- }
- return currentIterator.hasNext();
- }
-
- @Override
- public OzoneVolume next() {
- if(hasNext()) {
- currentValue = currentIterator.next();
- return currentValue;
- }
- throw new NoSuchElementException();
- }
-
- /**
- * Returns the next set of volume list using proxy.
- * @param prevVolume previous volume, this will be excluded from the result
- * @return {@code List<OzoneVolume>}
- */
- private List<OzoneVolume> getNextListOfVolumes(String prevVolume) {
- try {
- //if user is null, we do list of all volumes.
- if(user != null) {
- return proxy.listVolumes(user, volPrefix, prevVolume, listCacheSize);
- }
- return proxy.listVolumes(volPrefix, prevVolume, listCacheSize);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
deleted file mode 100644
index b94e0f7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * A class that encapsulates OzoneBucket.
- */
-public class OzoneBucket {
-
- /**
- * The proxy used for connecting to the cluster and perform
- * client operations.
- */
- private final ClientProtocol proxy;
- /**
- * Name of the volume in which the bucket belongs to.
- */
- private final String volumeName;
- /**
- * Name of the bucket.
- */
- private final String name;
- /**
- * Default replication factor to be used while creating keys.
- */
- private final ReplicationFactor defaultReplication;
-
- /**
- * Default replication type to be used while creating keys.
- */
- private final ReplicationType defaultReplicationType;
- /**
- * Bucket ACLs.
- */
- private List<OzoneAcl> acls;
-
- /**
- * Type of storage to be used for this bucket.
- * [RAM_DISK, SSD, DISK, ARCHIVE]
- */
- private StorageType storageType;
-
- /**
- * Bucket Version flag.
- */
- private Boolean versioning;
-
- /**
- * Cache size to be used for listKey calls.
- */
- private int listCacheSize;
-
- /**
- * Creation time of the bucket.
- */
- private long creationTime;
-
- /**
- * Constructs OzoneBucket instance.
- * @param conf Configuration object.
- * @param proxy ClientProtocol proxy.
- * @param volumeName Name of the volume the bucket belongs to.
- * @param bucketName Name of the bucket.
- * @param acls ACLs associated with the bucket.
- * @param storageType StorageType of the bucket.
- * @param versioning versioning status of the bucket.
- * @param creationTime creation time of the bucket.
- */
- public OzoneBucket(Configuration conf, ClientProtocol proxy,
- String volumeName, String bucketName,
- List<OzoneAcl> acls, StorageType storageType,
- Boolean versioning, long creationTime) {
- this.proxy = proxy;
- this.volumeName = volumeName;
- this.name = bucketName;
- this.acls = acls;
- this.storageType = storageType;
- this.versioning = versioning;
- this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
- this.creationTime = creationTime;
- this.defaultReplication = ReplicationFactor.valueOf(conf.getInt(
- OzoneConfigKeys.OZONE_REPLICATION,
- OzoneConfigKeys.OZONE_REPLICATION_DEFAULT));
- this.defaultReplicationType = ReplicationType.valueOf(conf.get(
- OzoneConfigKeys.OZONE_REPLICATION_TYPE,
- OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT));
- }
-
- /**
- * Returns Volume Name.
- *
- * @return volumeName
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Returns Bucket Name.
- *
- * @return bucketName
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns ACL's associated with the Bucket.
- *
- * @return acls
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Returns StorageType of the Bucket.
- *
- * @return storageType
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Returns Versioning associated with the Bucket.
- *
- * @return versioning
- */
- public Boolean getVersioning() {
- return versioning;
- }
-
- /**
- * Returns creation time of the Bucket.
- *
- * @return creation time of the bucket
- */
- public long getCreationTime() {
- return creationTime;
- }
-
- /**
- * Adds ACLs to the Bucket.
- * @param addAcls ACLs to be added
- * @throws IOException
- */
- public void addAcls(List<OzoneAcl> addAcls) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(addAcls);
- proxy.addBucketAcls(volumeName, name, addAcls);
- addAcls.stream().filter(acl -> !acls.contains(acl)).forEach(
- acls::add);
- }
-
- /**
- * Removes ACLs from the bucket.
- * @param removeAcls ACLs to be removed
- * @throws IOException
- */
- public void removeAcls(List<OzoneAcl> removeAcls) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(removeAcls);
- proxy.removeBucketAcls(volumeName, name, removeAcls);
- acls.removeAll(removeAcls);
- }
-
- /**
- * Sets/Changes the storage type of the bucket.
- * @param newStorageType Storage type to be set
- * @throws IOException
- */
- public void setStorageType(StorageType newStorageType) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(newStorageType);
- proxy.setBucketStorageType(volumeName, name, newStorageType);
- storageType = newStorageType;
- }
-
- /**
- * Enable/Disable versioning of the bucket.
- * @param newVersioning
- * @throws IOException
- */
- public void setVersioning(Boolean newVersioning) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(newVersioning);
- proxy.setBucketVersioning(volumeName, name, newVersioning);
- versioning = newVersioning;
- }
-
- /**
- * Creates a new key in the bucket, with default replication type RATIS and
- * with replication factor THREE.
- * @param key Name of the key to be created.
- * @param size Size of the data the key will point to.
- * @return OzoneOutputStream to which the data has to be written.
- * @throws IOException
- */
- public OzoneOutputStream createKey(String key, long size)
- throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(key);
- return createKey(key, size, defaultReplicationType, defaultReplication);
- }
-
- /**
- * Creates a new key in the bucket.
- * @param key Name of the key to be created.
- * @param size Size of the data the key will point to.
- * @param type Replication type to be used.
- * @param factor Replication factor of the key.
- * @return OzoneOutputStream to which the data has to be written.
- * @throws IOException
- */
- public OzoneOutputStream createKey(String key, long size,
- ReplicationType type,
- ReplicationFactor factor)
- throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(type);
- Preconditions.checkNotNull(factor);
- return proxy.createKey(volumeName, name, key, size, type, factor);
- }
-
- /**
- * Reads an existing key from the bucket.
- * @param key Name of the key to be read.
- * @return OzoneInputStream the stream using which the data can be read.
- * @throws IOException
- */
- public OzoneInputStream readKey(String key) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(key);
- return proxy.getKey(volumeName, name, key);
- }
-
- /**
- * Returns information about the key.
- * @param key Name of the key.
- * @return OzoneKey Information about the key.
- * @throws IOException
- */
- public OzoneKey getKey(String key) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(key);
- return proxy.getKeyDetails(volumeName, name, key);
- }
-
- /**
- * Returns Iterator to iterate over all keys in the bucket.
- * The result can be restricted using key prefix, will return all
- * keys if key prefix is null.
- *
- * @param keyPrefix Bucket prefix to match
- * @return {@code Iterator<OzoneKey>}
- */
- public Iterator<OzoneKey> listKeys(String keyPrefix) {
- return new KeyIterator(keyPrefix);
- }
-
- /**
- * Deletes key from the bucket.
- * @param key Name of the key to be deleted.
- * @throws IOException
- */
- public void deleteKey(String key) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(key);
- proxy.deleteKey(volumeName, name, key);
- }
-
- /**
- * An Iterator to iterate over {@link OzoneKey} list.
- */
- private class KeyIterator implements Iterator<OzoneKey> {
-
- private String keyPrefix = null;
-
- private Iterator<OzoneKey> currentIterator;
- private OzoneKey currentValue;
-
-
- /**
- * Creates an Iterator to iterate over all keys in the bucket,
- * which matches volume prefix.
- * @param keyPrefix
- */
- KeyIterator(String keyPrefix) {
- this.keyPrefix = keyPrefix;
- this.currentValue = null;
- this.currentIterator = getNextListOfKeys(null).iterator();
- }
-
- @Override
- public boolean hasNext() {
- if(!currentIterator.hasNext()) {
- currentIterator = getNextListOfKeys(
- currentValue != null ? currentValue.getName() : null)
- .iterator();
- }
- return currentIterator.hasNext();
- }
-
- @Override
- public OzoneKey next() {
- if(hasNext()) {
- currentValue = currentIterator.next();
- return currentValue;
- }
- throw new NoSuchElementException();
- }
-
- /**
- * Gets the next set of key list using proxy.
- * @param prevKey
- * @return {@code List<OzoneVolume>}
- */
- private List<OzoneKey> getNextListOfKeys(String prevKey) {
- try {
- return proxy.listKeys(volumeName, name, keyPrefix, prevKey,
- listCacheSize);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
deleted file mode 100644
index f191507..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClient.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * OzoneClient connects to Ozone Cluster and
- * perform basic operations.
- */
-public class OzoneClient implements Closeable {
-
- /*
- * OzoneClient connects to Ozone Cluster and
- * perform basic operations.
- *
- * +-------------+ +---+ +-------------------------------------+
- * | OzoneClient | --> | C | | Object Store |
- * |_____________| | l | | +-------------------------------+ |
- * | i | | | Volume(s) | |
- * | e | | | +------------------------+ | |
- * | n | | | | Bucket(s) | | |
- * | t | | | | +------------------+ | | |
- * | | | | | | Key -> Value (s) | | | |
- * | P |-->| | | | | | | |
- * | r | | | | |__________________| | | |
- * | o | | | | | | |
- * | t | | | |________________________| | |
- * | o | | | | |
- * | c | | |_______________________________| |
- * | o | | |
- * | l | |_____________________________________|
- * |___|
- * Example:
- * ObjectStore store = client.getObjectStore();
- * store.createVolume(“volume one”, VolumeArgs);
- * volume.setQuota(“10 GB”);
- * OzoneVolume volume = store.getVolume(“volume one”);
- * volume.createBucket(“bucket one”, BucketArgs);
- * bucket.setVersioning(true);
- * OzoneOutputStream os = bucket.createKey(“key one”, 1024);
- * os.write(byte[]);
- * os.close();
- * OzoneInputStream is = bucket.readKey(“key one”);
- * is.read();
- * is.close();
- * bucket.deleteKey(“key one”);
- * volume.deleteBucket(“bucket one”);
- * store.deleteVolume(“volume one”);
- * client.close();
- */
-
- private final ClientProtocol proxy;
- private final ObjectStore objectStore;
-
- /**
- * Creates a new OzoneClient object, generally constructed
- * using {@link OzoneClientFactory}.
- * @param conf Configuration object
- * @param proxy ClientProtocol proxy instance
- */
- public OzoneClient(Configuration conf, ClientProtocol proxy) {
- this.proxy = proxy;
- this.objectStore = new ObjectStore(conf, this.proxy);
- }
-
- /**
- * Returns the object store associated with the Ozone Cluster.
- * @return ObjectStore
- */
- public ObjectStore getObjectStore() {
- return objectStore;
- }
-
- /**
- * Closes the client and all the underlying resources.
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- proxy.close();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
deleted file mode 100644
index 292662e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
+++ /dev/null
@@ -1,599 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.DiskBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.Options;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
-import java.nio.channels.ReadableByteChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.UUID;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_KEEP_ALIVE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_MAX_POOL_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_THREAD_PRIORITY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT;
-
-/**
- * Class that writes to remote containers.
- */
-public class ContainerCacheFlusher implements Runnable {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerCacheFlusher.class);
- private final LinkedBlockingQueue<Message> messageQueue;
- private final ThreadPoolExecutor threadPoolExecutor;
- private final ArrayBlockingQueue<Runnable> workQueue;
- private final ConcurrentMap<String, RefCountedDB> dbMap;
- private final ByteBuffer blockIDBuffer;
- private final ConcurrentMap<String, Pipeline[]> pipelineMap;
- private final AtomicLong remoteIO;
- private final XceiverClientManager xceiverClientManager;
- private final CBlockTargetMetrics metrics;
- private AtomicBoolean shutdown;
- private final long levelDBCacheSize;
- private final int maxRetryCount;
- private final String tracePrefix;
-
- private final ConcurrentMap<String, FinishCounter> finishCountMap;
-
- /**
- * Constructs the writers to remote queue.
- */
- public ContainerCacheFlusher(Configuration config,
- XceiverClientManager xceiverClientManager,
- CBlockTargetMetrics metrics) {
- int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
- DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
- int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
- DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
- int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
- DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
- long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
- DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, TimeUnit.SECONDS);
- int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
- DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
- int blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
- DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
- levelDBCacheSize = config.getInt(DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY,
- DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT) * OzoneConsts.MB;
-
- LOG.info("Cache: Core Pool Size: {}", corePoolSize);
- LOG.info("Cache: Keep Alive: {}", keepAlive);
- LOG.info("Cache: Max Pool Size: {}", maxPoolSize);
- LOG.info("Cache: Thread Pri: {}", threadPri);
- LOG.info("Cache: BlockBuffer Size: {}", blockBufferSize);
-
- shutdown = new AtomicBoolean(false);
- messageQueue = new LinkedBlockingQueue<>();
- workQueue = new ArrayBlockingQueue<>(queueSize, true);
-
- ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat("Cache Block Writer Thread #%d")
- .setDaemon(true)
- .setPriority(threadPri)
- .build();
- threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize,
- keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
- new ThreadPoolExecutor.AbortPolicy());
- threadPoolExecutor.prestartAllCoreThreads();
-
- dbMap = new ConcurrentHashMap<>();
- pipelineMap = new ConcurrentHashMap<>();
- blockIDBuffer = ByteBuffer.allocateDirect(blockBufferSize);
- this.xceiverClientManager = xceiverClientManager;
- this.metrics = metrics;
- this.remoteIO = new AtomicLong();
-
- this.finishCountMap = new ConcurrentHashMap<>();
- this.maxRetryCount =
- config.getInt(CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_KEY,
- CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT);
- this.tracePrefix = getTracePrefix();
- }
-
- private void checkExistingLog(String prefixFileName, File dbPath) {
- if (!dbPath.exists()) {
- LOG.debug("No existing dirty log found at {}", dbPath);
- return;
- }
- LOG.debug("Need to check and requeue existing dirty log {}", dbPath);
- HashMap<String, ArrayList<String>> allFiles = new HashMap<>();
- traverse(prefixFileName, dbPath, allFiles);
- for (Map.Entry<String, ArrayList<String>> entry : allFiles.entrySet()) {
- String parentPath = entry.getKey();
- for (String fileName : entry.getValue()) {
- LOG.info("found {} {} with prefix {}",
- parentPath, fileName, prefixFileName);
- processDirtyBlocks(parentPath, fileName);
- }
- }
- }
-
- private void traverse(String prefixFileName, File path,
- HashMap<String, ArrayList<String>> files) {
- if (path.isFile()) {
- if (path.getName().startsWith(prefixFileName)) {
- LOG.debug("found this {} with {}", path.getParent(), path.getName());
- if (!files.containsKey(path.getParent())) {
- files.put(path.getParent(), new ArrayList<>());
- }
- files.get(path.getParent()).add(path.getName());
- }
- } else {
- File[] listFiles = path.listFiles();
- if (listFiles != null) {
- for (File subPath : listFiles) {
- traverse(prefixFileName, subPath, files);
- }
- }
- }
- }
-
- /**
- * Gets the CBlockTargetMetrics.
- *
- * @return CBlockTargetMetrics
- */
- public CBlockTargetMetrics getTargetMetrics() {
- return metrics;
- }
-
- /**
- * Gets the getXceiverClientManager.
- *
- * @return XceiverClientManager
- */
- public XceiverClientManager getXceiverClientManager() {
- return xceiverClientManager;
- }
-
- /**
- * Shutdown this instance.
- */
- public void shutdown() {
- this.shutdown.set(true);
- threadPoolExecutor.shutdown();
- }
-
- public long incrementRemoteIO() {
- return remoteIO.incrementAndGet();
- }
-
- /**
- * Processes a block cache file and queues those blocks for the remote I/O.
- *
- * @param dbPath - Location where the DB can be found.
- * @param fileName - Block Cache File Name
- */
- public void processDirtyBlocks(String dbPath, String fileName) {
- LOG.info("Adding {}/{} to queue. Queue Length: {}", dbPath, fileName,
- messageQueue.size());
- this.messageQueue.add(new Message(dbPath, fileName));
- }
-
- public Logger getLOG() {
- return LOG;
- }
-
- /**
- * Opens a DB if needed or returns a handle to an already open DB.
- *
- * @param dbPath -- dbPath
- * @return the levelDB on the given path.
- * @throws IOException
- */
- public synchronized LevelDBStore openDB(String dbPath)
- throws IOException {
- if (dbMap.containsKey(dbPath)) {
- RefCountedDB refDB = dbMap.get(dbPath);
- refDB.open();
- return refDB.db;
- } else {
- Options options = new Options();
- options.cacheSize(levelDBCacheSize);
- options.createIfMissing(true);
- LevelDBStore cacheDB = new LevelDBStore(
- new File(getDBFileName(dbPath)), options);
- RefCountedDB refDB = new RefCountedDB(dbPath, cacheDB);
- dbMap.put(dbPath, refDB);
- return cacheDB;
- }
- }
-
- /**
- * Updates the container map. This data never changes so we will update this
- * during restarts and it should not hurt us.
- *
- * Once a CBlockLocalCache cache is registered, requeue dirty/retry log files
- * for the volume
- *
- * @param dbPath - DbPath
- * @param containerList - Container List.
- */
- public void register(String dbPath, Pipeline[] containerList) {
- File dbFile = Paths.get(dbPath).toFile();
- pipelineMap.put(dbPath, containerList);
- checkExistingLog(AsyncBlockWriter.DIRTY_LOG_PREFIX, dbFile);
- checkExistingLog(AsyncBlockWriter.RETRY_LOG_PREFIX, dbFile);
- }
-
- private String getDBFileName(String dbPath) {
- return dbPath + ".db";
- }
-
- public LevelDBStore getCacheDB(String dbPath) throws IOException {
- return openDB(dbPath);
- }
-
- public void releaseCacheDB(String dbPath) {
- try {
- closeDB(dbPath);
- } catch (Exception e) {
- metrics.incNumFailedReleaseLevelDB();
- LOG.error("LevelDB close failed, dbPath:" + dbPath, e);
- }
- }
- /**
- * Close the DB if we don't have any outstanding references.
- *
- * @param dbPath - dbPath
- * @throws IOException
- */
- public synchronized void closeDB(String dbPath) throws IOException {
- if (dbMap.containsKey(dbPath)) {
- RefCountedDB refDB = dbMap.get(dbPath);
- int count = refDB.close();
- if (count == 0) {
- dbMap.remove(dbPath);
- }
- }
- }
-
- Pipeline getPipeline(String dbPath, long blockId) {
- Pipeline[] containerList = pipelineMap.get(dbPath);
- Preconditions.checkNotNull(containerList);
- int containerIdx = (int) blockId % containerList.length;
- long cBlockIndex =
- Longs.fromByteArray(containerList[containerIdx].getData());
- if (cBlockIndex > 0) {
- // This catches the case when we get a wrong container in the ordering
- // of the containers.
- Preconditions.checkState(containerIdx % cBlockIndex == 0,
- "The container ID computed should match with the container index " +
- "returned from cBlock Server.");
- }
- return containerList[containerIdx];
- }
-
- public void incFinishCount(String fileName) {
- if (!finishCountMap.containsKey(fileName)) {
- LOG.error("No record for such file:" + fileName);
- return;
- }
- finishCountMap.get(fileName).incCount();
- if (finishCountMap.get(fileName).isFileDeleted()) {
- finishCountMap.remove(fileName);
- }
- }
-
- /**
- * When an object implementing interface <code>Runnable</code> is used
- * to create a thread, starting the thread causes the object's
- * <code>run</code> method to be called in that separately executing
- * thread.
- * <p>
- * The general contract of the method <code>run</code> is that it may
- * take any action whatsoever.
- *
- * @see Thread#run()
- */
- @Override
- public void run() {
- while (!this.shutdown.get()) {
- try {
- Message message = messageQueue.take();
- LOG.debug("Got message to process -- DB Path : {} , FileName; {}",
- message.getDbPath(), message.getFileName());
- String fullPath = Paths.get(message.getDbPath(),
- message.getFileName()).toString();
- String[] fileNameParts = message.getFileName().split("\\.");
- Preconditions.checkState(fileNameParts.length > 1);
- String fileType = fileNameParts[0];
- boolean isDirtyLogFile =
- fileType.equalsIgnoreCase(AsyncBlockWriter.DIRTY_LOG_PREFIX);
- ReadableByteChannel fileChannel = new FileInputStream(fullPath)
- .getChannel();
- // TODO: We can batch and unique the IOs here. First getting the code
- // to work, we will add those later.
- int bytesRead = fileChannel.read(blockIDBuffer);
- fileChannel.close();
- LOG.debug("Read blockID log of size: {} position {} remaining {}",
- bytesRead, blockIDBuffer.position(), blockIDBuffer.remaining());
- // current position of in the buffer in bytes, divided by number of
- // bytes per long (which is calculated by number of bits per long
- // divided by number of bits per byte) gives the number of blocks
- int blockCount = blockIDBuffer.position()/(Long.SIZE / Byte.SIZE);
- if (isDirtyLogFile) {
- getTargetMetrics().incNumBytesDirtyLogRead(bytesRead);
- } else {
- getTargetMetrics().incNumBytesRetryLogRead(bytesRead);
- }
- if (finishCountMap.containsKey(message.getFileName())) {
- // In theory this should never happen. But if it happened,
- // we need to know it...
- getTargetMetrics().incNumIllegalDirtyLogFiles();
- LOG.error("Adding DirtyLog file again {} current count {} new {}",
- message.getFileName(),
- finishCountMap.get(message.getFileName()).expectedCount,
- blockCount);
- }
- finishCountMap.put(message.getFileName(),
- new FinishCounter(blockCount, message.getDbPath(),
- message.getFileName(), this));
- // should be flip instead of rewind, because we also need to make sure
- // the end position is correct.
- blockIDBuffer.flip();
- LOG.debug("Remaining blocks count {} and {}", blockIDBuffer.remaining(),
- blockCount);
- while (blockIDBuffer.remaining() >= (Long.SIZE / Byte.SIZE)) {
- long blockID = blockIDBuffer.getLong();
- int retryCount = 0;
- if (isDirtyLogFile) {
- getTargetMetrics().incNumDirtyLogBlockRead();
- } else {
- getTargetMetrics().incNumRetryLogBlockRead();
- Preconditions.checkState(fileNameParts.length == 4);
- retryCount = Integer.parseInt(fileNameParts[3]);
- }
- LogicalBlock block = new DiskBlock(blockID, null, false);
- BlockWriterTask blockWriterTask = new BlockWriterTask(block, this,
- message.getDbPath(), retryCount, message.getFileName(),
- maxRetryCount);
- threadPoolExecutor.submit(blockWriterTask);
- }
- blockIDBuffer.clear();
- } catch (InterruptedException e) {
- LOG.info("ContainerCacheFlusher is interrupted.", e);
- } catch (FileNotFoundException e) {
- LOG.error("Unable to find the dirty blocks file. This will cause " +
- "data errors. Please stop using this volume.", e);
- } catch (IOException e) {
- LOG.error("Unable to read the dirty blocks file. This will cause " +
- "data errors. Please stop using this volume.", e);
- } catch (Exception e) {
- LOG.error("Generic exception.", e);
- }
- }
- LOG.info("Exiting flusher");
- }
-
- /**
- * Tries to get the local host IP Address as trace prefix
- * for creating trace IDs, otherwise uses a random UUID for it.
- */
- private static String getTracePrefix() {
- String tmp;
- try {
- tmp = InetAddress.getLocalHost().getHostAddress();
- } catch (UnknownHostException ex) {
- tmp = UUID.randomUUID().toString();
- LOG.error("Unable to read the host address. Using a GUID for " +
- "hostname:{} ", tmp, ex);
- }
- return tmp;
- }
-
- /**
- * We create a trace ID to make it easy to debug issues.
- * A trace ID is in IPAddress:UserName:VolumeName:blockID:second format.
- *
- * This will get written down on the data node if we get any failures, so
- * with this trace ID we can correlate cBlock failures across machines.
- *
- * @param blockID - Block ID
- * @return trace ID
- */
- public String getTraceID(File dbPath, long blockID) {
- String volumeName = dbPath.getName();
- String userName = dbPath.getParentFile().getName();
- // mapping to seconds to make the string smaller.
- return tracePrefix + ":" + userName + ":" + volumeName
- + ":" + blockID + ":" + Time.monotonicNow() / 1000;
- }
-
- /**
- * Keeps a Reference counted DB that we close only when the total Reference
- * has gone to zero.
- */
- private static class RefCountedDB {
- private LevelDBStore db;
- private AtomicInteger refcount;
- private String dbPath;
-
- /**
- * RefCountedDB DB ctor.
- *
- * @param dbPath - DB path.
- * @param db - LevelDBStore db
- */
- RefCountedDB(String dbPath, LevelDBStore db) {
- this.db = db;
- this.refcount = new AtomicInteger(1);
- this.dbPath = dbPath;
- }
-
- /**
- * close the DB if possible.
- */
- public int close() throws IOException {
- int count = this.refcount.decrementAndGet();
- if (count == 0) {
- LOG.info("Closing the LevelDB. {} ", this.dbPath);
- db.close();
- }
- return count;
- }
-
- public void open() {
- this.refcount.incrementAndGet();
- }
- }
-
- /**
- * The message held in processing queue.
- */
- private static class Message {
- private String dbPath;
- private String fileName;
-
- /**
- * A message that holds the info about which path dirty blocks log and
- * which path contains db.
- *
- * @param dbPath
- * @param fileName
- */
- Message(String dbPath, String fileName) {
- this.dbPath = dbPath;
- this.fileName = fileName;
- }
-
- public String getDbPath() {
- return dbPath;
- }
-
- public void setDbPath(String dbPath) {
- this.dbPath = dbPath;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public void setFileName(String fileName) {
- this.fileName = fileName;
- }
- }
-
- private static class FinishCounter {
- private final long expectedCount;
- private final String dbPath;
- private final String dirtyLogPath;
- private final AtomicLong currentCount;
- private AtomicBoolean fileDeleted;
- private final ContainerCacheFlusher flusher;
-
- FinishCounter(long expectedCount, String dbPath,
- String dirtyLogPath, ContainerCacheFlusher flusher) throws IOException {
- this.expectedCount = expectedCount;
- this.dbPath = dbPath;
- this.dirtyLogPath = dirtyLogPath;
- this.currentCount = new AtomicLong(0);
- this.fileDeleted = new AtomicBoolean(false);
- this.flusher = flusher;
- }
-
- public boolean isFileDeleted() {
- return fileDeleted.get();
- }
-
- public void incCount() {
- long count = this.currentCount.incrementAndGet();
- if (count >= expectedCount) {
- String filePath = String.format("%s/%s", dbPath, dirtyLogPath);
- LOG.debug(
- "Deleting {} with count {} {}", filePath, count, expectedCount);
- try {
- Path path = Paths.get(filePath);
- Files.delete(path);
- // the following part tries to remove the directory if it is empty
- // but not sufficient, because the .db directory still exists....
- // TODO how to handle the .db directory?
- /*Path parent = path.getParent();
- if (parent.toFile().listFiles().length == 0) {
- Files.delete(parent);
- }*/
- fileDeleted.set(true);
- } catch (Exception e) {
- flusher.getTargetMetrics().incNumFailedDirtyLogFileDeletes();
- LOG.error("Error deleting dirty log file:" + filePath, e);
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
deleted file mode 100644
index ce7717b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.jscsi.target.Configuration;
-
-import java.net.InetSocketAddress;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
-
-/**
- * This class runs the target server process.
- */
-public final class SCSITargetDaemon {
- public static void main(String[] args) throws Exception {
- OzoneConfiguration ozoneConf = new OzoneConfiguration();
-
- RPC.setProtocolEngine(ozoneConf, CBlockClientServerProtocolPB.class,
- ProtobufRpcEngine.class);
- long containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
- DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
- ContainerOperationClient.setContainerSizeB(
- containerSizeGB * OzoneConsts.GB);
- String jscsiServerAddress = ozoneConf.get(
- DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
- DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
- String cbmIPAddress = ozoneConf.get(
- DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
- DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
- );
- int cbmPort = ozoneConf.getInt(
- DFS_CBLOCK_JSCSI_PORT_KEY,
- DFS_CBLOCK_JSCSI_PORT_DEFAULT
- );
-
- String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
- OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
- int scmClientPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
- OZONE_SCM_CLIENT_PORT_DEFAULT);
- int scmDatanodePort = ozoneConf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
- OZONE_SCM_DATANODE_PORT_DEFAULT);
-
- String scmClientAddress = scmAddress + ":" + scmClientPort;
- String scmDataodeAddress = scmAddress + ":" + scmDatanodePort;
-
- ozoneConf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmClientAddress);
- ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
-
- InetSocketAddress cbmAddress = new InetSocketAddress(
- cbmIPAddress, cbmPort);
- long version = RPC.getProtocolVersion(
- CBlockServiceProtocolPB.class);
- CBlockClientProtocolClientSideTranslatorPB cbmClient =
- new CBlockClientProtocolClientSideTranslatorPB(
- RPC.getProxy(CBlockClientServerProtocolPB.class, version,
- cbmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
- NetUtils.getDefaultSocketFactory(ozoneConf), 5000)
- );
- CBlockManagerHandler cbmHandler = new CBlockManagerHandler(cbmClient);
-
- String advertisedAddress = ozoneConf.
- getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
-
- int advertisedPort = ozoneConf.
- getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
- DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
-
- Configuration jscsiConfig =
- new Configuration(jscsiServerAddress,
- advertisedAddress,
- advertisedPort);
- DefaultMetricsSystem.initialize("CBlockMetrics");
- CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
- CBlockTargetServer targetServer = new CBlockTargetServer(
- ozoneConf, jscsiConfig, cbmHandler, metrics);
-
- targetServer.call();
- }
-
- private SCSITargetDaemon() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
deleted file mode 100644
index 300b2ae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;
-
-import java.io.IOException;
-
-/**
- * Defines the interface for cache implementations. The cache will be called
- * by cblock storage module when it performs IO operations.
- */
-public interface CacheModule {
- /**
- * check if the key is cached, if yes, returned the cached object.
- * otherwise, load from data source. Then put it into cache.
- *
- * @param blockID
- * @return the target block.
- */
- LogicalBlock get(long blockID) throws IOException;
-
- /**
- * put the value of the key into cache.
- * @param blockID
- * @param value
- */
- void put(long blockID, byte[] value) throws IOException;
-
- void flush() throws IOException;
-
- void start() throws IOException;
-
- void stop() throws IOException;
-
- void close() throws IOException;
-
- boolean isDirtyCache();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
deleted file mode 100644
index 470826f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;
-
-import java.nio.ByteBuffer;
-
-/**
- * Logical Block is the data structure that we write to the cache,
- * the key and data gets written to remote contianers. Rest is used for
- * book keeping for the cache.
- */
-public interface LogicalBlock {
- /**
- * Returns the data stream of this block.
- * @return - ByteBuffer
- */
- ByteBuffer getData();
-
- /**
- * Frees the byte buffer since we don't need it any more.
- */
- void clearData();
-
- /**
- * Returns the Block ID for this Block.
- * @return long - BlockID
- */
- long getBlockID();
-
- /**
- * Flag that tells us if this block has been persisted to container.
- * @return whether this block is now persistent
- */
- boolean isPersisted();
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
deleted file mode 100644
index 992578f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
+++ /dev/null
@@ -1,221 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Condition;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * A Queue that is used to write blocks asynchronously to the container.
- */
-public class AsyncBlockWriter {
- private static final Logger LOG =
- LoggerFactory.getLogger(AsyncBlockWriter.class);
-
- /**
- * XceiverClientManager is used to get client connections to a set of
- * machines.
- */
- private final XceiverClientManager xceiverClientManager;
-
- /**
- * This lock is used as a signal to re-queuing thread. The requeue thread
- * wakes up as soon as it is signaled some blocks are in the retry queue.
- * We try really aggressively since this new block will automatically move
- * to the end of the queue.
- * <p>
- * In the event a container is unavailable for a long time, we can either
- * fail all writes or remap and let the writes succeed. The easier
- * semantics is to fail the volume until the container is recovered by SCM.
- */
- private final Lock lock;
- private final Condition notEmpty;
- /**
- * The cache this writer is operating against.
- */
- private final CBlockLocalCache parentCache;
- private final BlockBufferManager blockBufferManager;
- public final static String DIRTY_LOG_PREFIX = "DirtyLog";
- public static final String RETRY_LOG_PREFIX = "RetryLog";
- private AtomicLong localIoCount;
-
- /**
- * Constructs an Async Block Writer.
- *
- * @param config - Config
- * @param cache - Parent Cache for this writer
- */
- public AsyncBlockWriter(Configuration config, CBlockLocalCache cache) {
-
- Preconditions.checkNotNull(cache, "Cache cannot be null.");
- Preconditions.checkNotNull(cache.getCacheDB(), "DB cannot be null.");
- localIoCount = new AtomicLong();
- lock = new ReentrantLock();
- notEmpty = lock.newCondition();
- parentCache = cache;
- xceiverClientManager = cache.getClientManager();
- blockBufferManager = new BlockBufferManager(config, parentCache);
- }
-
- public void start() throws IOException {
- File logDir = new File(parentCache.getDbPath().toString());
- if (!logDir.exists() && !logDir.mkdirs()) {
- LOG.error("Unable to create the log directory, Critical error cannot " +
- "continue. Log Dir : {}", logDir);
- throw new IllegalStateException("Cache Directory create failed, Cannot " +
- "continue. Log Dir: {}" + logDir);
- }
- blockBufferManager.start();
- }
-
- /**
- * Return the log to write to.
- *
- * @return Logger.
- */
- public static Logger getLOG() {
- return LOG;
- }
-
- /**
- * Get the CacheDB.
- *
- * @return LevelDB Handle
- */
- LevelDBStore getCacheDB() {
- return parentCache.getCacheDB();
- }
-
- /**
- * Returns the client manager.
- *
- * @return XceiverClientManager
- */
- XceiverClientManager getXceiverClientManager() {
- return xceiverClientManager;
- }
-
- /**
- * Incs the localIoPacket Count that has gone into this device.
- */
- public long incrementLocalIO() {
- return localIoCount.incrementAndGet();
- }
-
- /**
- * Return the local io counts to this device.
- * @return the count of io
- */
- public long getLocalIOCount() {
- return localIoCount.get();
- }
-
- /**
- * Writes a block to LevelDB store and queues a work item for the system to
- * sync the block to containers.
- *
- * @param block - Logical Block
- */
- public void writeBlock(LogicalBlock block) throws IOException {
- byte[] keybuf = Longs.toByteArray(block.getBlockID());
- String traceID = parentCache.getTraceID(block.getBlockID());
- if (parentCache.isShortCircuitIOEnabled()) {
- long startTime = Time.monotonicNow();
- getCacheDB().put(keybuf, block.getData().array());
- incrementLocalIO();
- long endTime = Time.monotonicNow();
- parentCache.getTargetMetrics().updateDBWriteLatency(
- endTime - startTime);
- if (parentCache.isTraceEnabled()) {
- String datahash = DigestUtils.sha256Hex(block.getData().array());
- parentCache.getTracer().info(
- "Task=WriterTaskDBPut,BlockID={},Time={},SHA={}",
- block.getBlockID(), endTime - startTime, datahash);
- }
- block.clearData();
- blockBufferManager.addToBlockBuffer(block.getBlockID());
- } else {
- Pipeline pipeline = parentCache.getPipeline(block.getBlockID());
- String containerName = pipeline.getContainerName();
- XceiverClientSpi client = null;
- try {
- long startTime = Time.monotonicNow();
- client = parentCache.getClientManager()
- .acquireClient(parentCache.getPipeline(block.getBlockID()));
- ContainerProtocolCalls.writeSmallFile(client, containerName,
- Long.toString(block.getBlockID()), block.getData().array(),
- traceID);
- long endTime = Time.monotonicNow();
- if (parentCache.isTraceEnabled()) {
- String datahash = DigestUtils.sha256Hex(block.getData().array());
- parentCache.getTracer().info(
- "Task=DirectWriterPut,BlockID={},Time={},SHA={}",
- block.getBlockID(), endTime - startTime, datahash);
- }
- parentCache.getTargetMetrics().
- updateDirectBlockWriteLatency(endTime - startTime);
- parentCache.getTargetMetrics().incNumDirectBlockWrites();
- } catch (Exception ex) {
- parentCache.getTargetMetrics().incNumFailedDirectBlockWrites();
- LOG.error("Direct I/O writing of block:{} traceID:{} to "
- + "container {} failed", block.getBlockID(), traceID,
- containerName, ex);
- throw ex;
- } finally {
- if (client != null) {
- parentCache.getClientManager().releaseClient(client);
- }
- block.clearData();
- }
- }
- }
-
- /**
- * Shutdown by writing any pending I/O to dirtylog buffer.
- */
- public void shutdown() {
- blockBufferManager.shutdown();
- }
- /**
- * Returns tracer.
- *
- * @return Tracer
- */
- Logger getTracer() {
- return parentCache.getTracer();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
deleted file mode 100644
index c61a7a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
-
-/**
- * This task is responsible for flushing the BlockIDBuffer
- * to Dirty Log File. This Dirty Log file is used later by
- * ContainerCacheFlusher when the data is written to container
- */
-public class BlockBufferFlushTask implements Runnable {
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockBufferFlushTask.class);
- private final CBlockLocalCache parentCache;
- private final BlockBufferManager bufferManager;
- private final ByteBuffer blockIDBuffer;
-
- BlockBufferFlushTask(ByteBuffer blockIDBuffer, CBlockLocalCache parentCache,
- BlockBufferManager manager) {
- this.parentCache = parentCache;
- this.bufferManager = manager;
- this.blockIDBuffer = blockIDBuffer;
- }
-
- /**
- * When an object implementing interface <code>Runnable</code> is used
- * to create a thread, starting the thread causes the object's
- * <code>run</code> method to be called in that separately executing
- * thread.
- * <p>
- * The general contract of the method <code>run</code> is that it may
- * take any action whatsoever.
- *
- * @see Thread#run()
- */
- @Override
- public void run() {
- try {
- writeBlockBufferToFile(blockIDBuffer);
- } catch (Exception e) {
- parentCache.getTargetMetrics().incNumFailedBlockBufferFlushes();
- LOG.error("Unable to sync the Block map to disk with "
- + (blockIDBuffer.position() / Long.SIZE) + "entries "
- + "-- NOTE: This might cause a data loss or corruption", e);
- } finally {
- bufferManager.releaseBuffer(blockIDBuffer);
- }
- }
-
- /**
- * Write Block Buffer to file.
- *
- * @param buffer - ByteBuffer
- * @throws IOException
- */
- private void writeBlockBufferToFile(ByteBuffer buffer)
- throws IOException {
- long startTime = Time.monotonicNow();
- boolean append = false;
-
- // If there is nothing written to blockId buffer,
- // then skip flushing of blockId buffer
- if (buffer.position() == 0) {
- return;
- }
-
- buffer.flip();
- String fileName =
- String.format("%s.%s", AsyncBlockWriter.DIRTY_LOG_PREFIX,
- Time.monotonicNow());
- String log = Paths.get(parentCache.getDbPath().toString(), fileName)
- .toString();
-
- FileChannel channel = new FileOutputStream(log, append).getChannel();
- int bytesWritten = channel.write(buffer);
- channel.close();
- buffer.clear();
- parentCache.processDirtyMessage(fileName);
- long endTime = Time.monotonicNow();
- if (parentCache.isTraceEnabled()) {
- parentCache.getTracer().info(
- "Task=DirtyBlockLogWrite,Time={} bytesWritten={}",
- endTime - startTime, bytesWritten);
- }
-
- parentCache.getTargetMetrics().incNumBlockBufferFlushCompleted();
- parentCache.getTargetMetrics().incNumBytesDirtyLogWritten(bytesWritten);
- parentCache.getTargetMetrics().
- updateBlockBufferFlushLatency(endTime - startTime);
- LOG.debug("Block buffer writer bytesWritten:{} Time:{}",
- bytesWritten, endTime - startTime);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
deleted file mode 100644
index 5d3209c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ScheduledThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ThreadFactory;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_KEEP_ALIVE;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_THREAD_PRIORITY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.
- DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
-
-/**
- * This class manages the block ID buffer.
- * Block ID Buffer keeps a list of blocks which are in leveldb cache
- * This buffer is used later when the blocks are flushed to container
- *
- * Two blockIDBuffers are maintained so that write are not blocked when
- * DirtyLog is being written. Once a blockIDBuffer is full, it will be
- * enqueued for DirtyLog write while the other buffer accepts new write.
- * Once the DirtyLog write is done, the buffer is returned back to the pool.
- *
- * There are three triggers for blockIDBuffer flush
- * 1) BlockIDBuffer is full,
- * 2) Time period defined for blockIDBuffer flush has elapsed.
- * 3) Shutdown
- */
-public class BlockBufferManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockBufferManager.class);
-
- private enum FlushReason {
- BUFFER_FULL,
- SHUTDOWN,
- TIMER
- };
-
- private final int blockBufferSize;
- private final CBlockLocalCache parentCache;
- private final ScheduledThreadPoolExecutor scheduledExecutor;
- private final ThreadPoolExecutor threadPoolExecutor;
- private final long intervalSeconds;
- private final ArrayBlockingQueue<ByteBuffer> acquireQueue;
- private final ArrayBlockingQueue<Runnable> workQueue;
- private ByteBuffer currentBuffer;
-
- BlockBufferManager(Configuration config, CBlockLocalCache parentCache) {
- this.parentCache = parentCache;
- this.scheduledExecutor = new ScheduledThreadPoolExecutor(1);
-
- this.intervalSeconds =
- config.getTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
- DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT,
- TimeUnit.SECONDS);
-
- long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
- DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT,
- TimeUnit.SECONDS);
- this.workQueue = new ArrayBlockingQueue<>(2, true);
- int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
- DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
- ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat("Cache Block Buffer Manager Thread #%d")
- .setDaemon(true)
- .setPriority(threadPri)
- .build();
- /*
- * starting a thread pool with core pool size of 1 and maximum of 2 threads
- * as there are maximum of 2 buffers which can be flushed at the same time.
- */
- this.threadPoolExecutor = new ThreadPoolExecutor(1, 2,
- keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
- new ThreadPoolExecutor.AbortPolicy());
-
- this.blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
- DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
- this.acquireQueue = new ArrayBlockingQueue<>(2, true);
-
- for (int i = 0; i < 2; i++) {
- acquireQueue.add(ByteBuffer.allocate(blockBufferSize));
- }
- // get the first buffer to be used
- this.currentBuffer = acquireQueue.remove();
-
- LOG.info("BufferManager: Buffer Size:{} FlushIntervalSeconds:{}",
- blockBufferSize, intervalSeconds);
- }
-
- // triggerBlockBufferFlush enqueues current ByteBuffer for flush and returns.
- // This enqueue is asynchronous and hence triggerBlockBufferFlush will
- // only block when there are no available buffers in acquireQueue
- // Once the DirtyLog write is done, buffer is returned back to
- // BlockBufferManager using releaseBuffer
- private synchronized void triggerBlockBufferFlush(FlushReason reason) {
- LOG.debug("Flush triggered because: " + reason.toString() +
- " Num entries in buffer: " +
- currentBuffer.position() / (Long.SIZE / Byte.SIZE) +
- " Acquire Queue Size: " + acquireQueue.size());
-
- parentCache.getTargetMetrics().incNumBlockBufferFlushTriggered();
- BlockBufferFlushTask flushTask =
- new BlockBufferFlushTask(currentBuffer, parentCache, this);
- threadPoolExecutor.submit(flushTask);
- try {
- currentBuffer = acquireQueue.take();
- } catch (InterruptedException ex) {
- currentBuffer = null;
- parentCache.getTargetMetrics().incNumInterruptedBufferWaits();
- LOG.error("wait on take operation on acquire queue interrupted", ex);
- Thread.currentThread().interrupt();
- }
- }
-
- public synchronized void addToBlockBuffer(long blockId) {
- parentCache.getTargetMetrics().incNumBlockBufferUpdates();
- currentBuffer.putLong(blockId);
- // if no space left, flush this buffer
- if (currentBuffer.remaining() == 0) {
- triggerBlockBufferFlush(FlushReason.BUFFER_FULL);
- }
- }
-
- public void releaseBuffer(ByteBuffer buffer) {
- if (buffer.position() != 0) {
- LOG.error("requeuing a non empty buffer with:{}",
- "elements enqueued in the acquire queue",
- buffer.position() / (Long.SIZE / Byte.SIZE));
- buffer.reset();
- }
- // There should always be space in the queue to add an element
- acquireQueue.add(buffer);
- }
-
- // Start a scheduled task to flush blockIDBuffer
- public void start() {
- Runnable scheduledTask = () -> triggerBlockBufferFlush(FlushReason.TIMER);
- scheduledExecutor.scheduleWithFixedDelay(scheduledTask, intervalSeconds,
- intervalSeconds, TimeUnit.SECONDS);
- threadPoolExecutor.prestartAllCoreThreads();
- }
-
- public void shutdown() {
- triggerBlockBufferFlush(FlushReason.SHUTDOWN);
- scheduledExecutor.shutdown();
- threadPoolExecutor.shutdown();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
deleted file mode 100644
index 1149164..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
-import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.FileStore;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_TRACE_IO_DEFAULT;
-
-/**
- * A local cache used by the CBlock ISCSI server. This class is enabled or
- * disabled via config settings.
- */
-public class CBlockLocalCache implements CacheModule {
- private static final Logger LOG =
- LoggerFactory.getLogger(CBlockLocalCache.class);
- private static final Logger TRACER =
- LoggerFactory.getLogger("TraceIO");
-
- private final Configuration conf;
- /**
- * LevelDB cache file.
- */
- private final LevelDBStore cacheDB;
-
- /**
- * AsyncBlock writer updates the cacheDB and writes the blocks async to
- * remote containers.
- */
- private final AsyncBlockWriter blockWriter;
-
- /**
- * Sync block reader tries to read from the cache and if we get a cache
- * miss we will fetch the block from remote location. It will asynchronously
- * update the cacheDB.
- */
- private final SyncBlockReader blockReader;
- private final String userName;
- private final String volumeName;
-
- /**
- * From a block ID we are able to get the pipeline by indexing this array.
- */
- private final Pipeline[] containerList;
- private final int blockSize;
- private XceiverClientManager clientManager;
- /**
- * If this flag is enabled then cache traces all I/O, all reads and writes
- * are visible in the log with sha of the block written. Makes the system
- * slower use it only for debugging or creating trace simulations.
- */
- private final boolean traceEnabled;
- private final boolean enableShortCircuitIO;
- private final long volumeSize;
- private long currentCacheSize;
- private File dbPath;
- private final ContainerCacheFlusher flusher;
- private CBlockTargetMetrics cblockTargetMetrics;
-
- /**
- * Get Db Path.
- * @return the file instance of the db.
- */
- public File getDbPath() {
- return dbPath;
- }
-
- /**
- * Constructor for CBlockLocalCache invoked via the builder.
- *
- * @param conf - Configuration
- * @param volumeName - volume Name
- * @param userName - user name
- * @param containerPipelines - Pipelines that make up this contianer
- * @param blockSize - blockSize
- * @param flusher - flusher to flush data to container
- * @throws IOException
- */
- CBlockLocalCache(
- Configuration conf, String volumeName,
- String userName, List<Pipeline> containerPipelines, int blockSize,
- long volumeSize, ContainerCacheFlusher flusher) throws IOException {
- this.conf = conf;
- this.userName = userName;
- this.volumeName = volumeName;
- this.blockSize = blockSize;
- this.flusher = flusher;
- this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
- DFS_CBLOCK_TRACE_IO_DEFAULT);
- this.enableShortCircuitIO = conf.getBoolean(
- DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO,
- DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT);
- dbPath = Paths.get(conf.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
- DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT), userName, volumeName).toFile();
-
- if (!dbPath.exists() && !dbPath.mkdirs()) {
- LOG.error("Unable to create the cache paths. Path: {}", dbPath);
- throw new IllegalArgumentException("Unable to create paths. Path: " +
- dbPath);
- }
- cacheDB = flusher.getCacheDB(dbPath.toString());
- this.containerList = containerPipelines.toArray(new
- Pipeline[containerPipelines.size()]);
- this.volumeSize = volumeSize;
-
- blockWriter = new AsyncBlockWriter(conf, this);
- blockReader = new SyncBlockReader(conf, this);
- if (this.traceEnabled) {
- getTracer().info("Task=StartingCache");
- }
- }
-
- private void setClientManager(XceiverClientManager manager) {
- this.clientManager = manager;
- }
-
- private void setCblockTargetMetrics(CBlockTargetMetrics targetMetrics) {
- this.cblockTargetMetrics = targetMetrics;
- }
-
- /**
- * Returns new builder class that builds a CBlockLocalCache.
- *
- * @return Builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- public void processDirtyMessage(String fileName) {
- flusher.processDirtyBlocks(dbPath.toString(), fileName);
- }
-
- /**
- * Get usable disk space.
- *
- * @param dbPathString - Path to db
- * @return long bytes remaining.
- */
- private static long getRemainingDiskSpace(String dbPathString) {
- try {
- URI fileUri = new URI("file:///");
- Path dbPath = Paths.get(fileUri).resolve(dbPathString);
- FileStore disk = Files.getFileStore(dbPath);
- return disk.getUsableSpace();
- } catch (URISyntaxException | IOException ex) {
- LOG.error("Unable to get free space on for path :" + dbPathString);
- }
- return 0L;
- }
-
- /**
- * Returns the Max current CacheSize.
- *
- * @return - Cache Size
- */
- public long getCurrentCacheSize() {
- return currentCacheSize;
- }
-
- /**
- * Sets the Maximum Cache Size.
- *
- * @param currentCacheSize - Max current Cache Size.
- */
- public void setCurrentCacheSize(long currentCacheSize) {
- this.currentCacheSize = currentCacheSize;
- }
-
- /**
- * True if block tracing is enabled.
- *
- * @return - bool
- */
- public boolean isTraceEnabled() {
- return traceEnabled;
- }
-
- /**
- * Checks if Short Circuit I/O is enabled.
- *
- * @return - true if it is enabled.
- */
- public boolean isShortCircuitIOEnabled() {
- return enableShortCircuitIO;
- }
-
- /**
- * Returns the default block size of this device.
- *
- * @return - int
- */
- public int getBlockSize() {
- return blockSize;
- }
-
- /**
- * Gets the client manager.
- *
- * @return XceiverClientManager
- */
- public XceiverClientManager getClientManager() {
- return clientManager;
- }
-
- /**
- * check if the key is cached, if yes, returned the cached object.
- * otherwise, load from data source. Then put it into cache.
- *
- * @param blockID
- * @return the block associated to the blockID
- */
- @Override
- public LogicalBlock get(long blockID) throws IOException {
- cblockTargetMetrics.incNumReadOps();
- return blockReader.readBlock(blockID);
- }
-
- /**
- * put the value of the key into cache and remote container.
- *
- * @param blockID - BlockID
- * @param data - byte[]
- */
- @Override
- public void put(long blockID, byte[] data) throws IOException {
- cblockTargetMetrics.incNumWriteOps();
- LogicalBlock block = new DiskBlock(blockID, data, false);
- blockWriter.writeBlock(block);
- }
-
- @Override
- public void flush() throws IOException {
-
- }
-
- @Override
- public void start() throws IOException {
- flusher.register(getDbPath().getPath(), containerList);
- blockWriter.start();
- }
-
- @Override
- public void stop() throws IOException {
- }
-
- @Override
- public void close() throws IOException {
- blockReader.shutdown();
- blockWriter.shutdown();
- this.flusher.releaseCacheDB(dbPath.toString());
- if (this.traceEnabled) {
- getTracer().info("Task=ShutdownCache");
- }
- }
-
- /**
- * Returns true if cache still has blocks pending to write.
- *
- * @return false if we have no pending blocks to write.
- */
- @Override
- public boolean isDirtyCache() {
- return false;
- }
-
- /**
- * Returns the local cache DB.
- *
- * @return - DB
- */
- LevelDBStore getCacheDB() {
- return this.cacheDB;
- }
-
- /**
- * Returns the current userName.
- *
- * @return - UserName
- */
- String getUserName() {
- return this.userName;
- }
-
- /**
- * Returns the volume name.
- *
- * @return VolumeName.
- */
- String getVolumeName() {
- return this.volumeName;
- }
-
- /**
- * Returns the target metrics.
- *
- * @return CBlock Target Metrics.
- */
- CBlockTargetMetrics getTargetMetrics() {
- return this.cblockTargetMetrics;
- }
-
- /**
- * Returns the pipeline to use given a container.
- *
- * @param blockId - blockID
- * @return - pipeline.
- */
- Pipeline getPipeline(long blockId) {
- int containerIdx = (int) blockId % containerList.length;
- long cBlockIndex =
- Longs.fromByteArray(containerList[containerIdx].getData());
- if (cBlockIndex > 0) {
- // This catches the case when we get a wrong container in the ordering
- // of the containers.
- Preconditions.checkState(containerIdx % cBlockIndex == 0,
- "The container ID computed should match with the container index " +
- "returned from cBlock Server.");
- }
- return containerList[containerIdx];
- }
-
- String getTraceID(long blockID) {
- return flusher.getTraceID(dbPath, blockID);
- }
-
- /**
- * Returns tracer.
- *
- * @return - Logger
- */
- Logger getTracer() {
- return TRACER;
- }
-
- /**
- * Builder class for CBlocklocalCache.
- */
- public static class Builder {
- private Configuration configuration;
- private String userName;
- private String volumeName;
- private List<Pipeline> pipelines;
- private XceiverClientManager clientManager;
- private int blockSize;
- private long volumeSize;
- private ContainerCacheFlusher flusher;
- private CBlockTargetMetrics metrics;
-
- /**
- * Ctor.
- */
- Builder() {
- }
-
- /**
- * Computes a cache size based on the configuration and available disk
- * space.
- *
- * @param configuration - Config
- * @param volumeSize - Size of Volume
- * @param blockSize - Size of the block
- * @return - cache size in bytes.
- */
- private static long computeCacheSize(Configuration configuration,
- long volumeSize, int blockSize) {
- long cacheSize = 0;
- String dbPath = configuration.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
- DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT);
- if (StringUtils.isBlank(dbPath)) {
- return cacheSize;
- }
- long spaceRemaining = getRemainingDiskSpace(dbPath);
- double cacheRatio = 1.0;
-
- if (spaceRemaining < volumeSize) {
- cacheRatio = (double)spaceRemaining / volumeSize;
- }
-
- // if cache is going to be at least 10% of the volume size it is worth
- // doing, otherwise skip creating the cache.
- if (cacheRatio >= 0.10) {
- cacheSize = Double.doubleToLongBits(volumeSize * cacheRatio);
- }
- return cacheSize;
- }
-
- /**
- * Sets the Config to be used by this cache.
- *
- * @param conf - Config
- * @return Builder
- */
- public Builder setConfiguration(Configuration conf) {
- this.configuration = conf;
- return this;
- }
-
- /**
- * Sets the user name who is the owner of this volume.
- *
- * @param user - name of the owner, please note this is not the current
- * user name.
- * @return - Builder
- */
- public Builder setUserName(String user) {
- this.userName = user;
- return this;
- }
-
- /**
- * Sets the VolumeName.
- *
- * @param volume - Name of the volume
- * @return Builder
- */
- public Builder setVolumeName(String volume) {
- this.volumeName = volume;
- return this;
- }
-
- /**
- * Sets the Pipelines that form this volume.
- *
- * @param pipelineList - list of pipelines
- * @return Builder
- */
- public Builder setPipelines(List<Pipeline> pipelineList) {
- this.pipelines = pipelineList;
- return this;
- }
-
- /**
- * Sets the Client Manager that manages the communication with containers.
- *
- * @param xceiverClientManager - clientManager.
- * @return - Builder
- */
- public Builder setClientManager(XceiverClientManager xceiverClientManager) {
- this.clientManager = xceiverClientManager;
- return this;
- }
-
- /**
- * Sets the block size -- Typical sizes are 4KB, 8KB etc.
- *
- * @param size - BlockSize.
- * @return - Builder
- */
- public Builder setBlockSize(int size) {
- this.blockSize = size;
- return this;
- }
-
- /**
- * Sets the volumeSize.
- *
- * @param size - VolumeSize
- * @return - Builder
- */
- public Builder setVolumeSize(long size) {
- this.volumeSize = size;
- return this;
- }
-
- /**
- * Set flusher.
- * @param containerCacheFlusher - cache Flusher
- * @return Builder.
- */
- public Builder setFlusher(ContainerCacheFlusher containerCacheFlusher) {
- this.flusher = containerCacheFlusher;
- return this;
- }
-
- /**
- * Sets the cblock Metrics.
- *
- * @param targetMetrics - CBlock Target Metrics
- * @return - Builder
- */
- public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
- this.metrics = targetMetrics;
- return this;
- }
-
- /**
- * Constructs a CBlockLocalCache.
- *
- * @return the CBlockLocalCache with the preset properties.
- * @throws IOException
- */
- public CBlockLocalCache build() throws IOException {
- Preconditions.checkNotNull(this.configuration, "A valid configuration " +
- "is needed");
- Preconditions.checkState(StringUtils.isNotBlank(userName), "A valid " +
- "username is needed");
- Preconditions.checkState(StringUtils.isNotBlank(volumeName), " A valid" +
- " volume name is needed");
- Preconditions.checkNotNull(this.pipelines, "Pipelines cannot be null");
- Preconditions.checkState(this.pipelines.size() > 0, "At least one " +
- "pipeline location is needed for a volume");
-
- for (int x = 0; x < pipelines.size(); x++) {
- Preconditions.checkNotNull(pipelines.get(x).getData(), "cBlock " +
- "relies on private data on the pipeline, null data found.");
- }
-
- Preconditions.checkNotNull(clientManager, "Client Manager cannot be " +
- "null");
- Preconditions.checkState(blockSize > 0, " Block size has to be a " +
- "number greater than 0");
-
- Preconditions.checkState(volumeSize > 0, "Volume Size cannot be less " +
- "than 1");
- Preconditions.checkNotNull(this.flusher, "Flusher cannot be null.");
-
- CBlockLocalCache cache = new CBlockLocalCache(this.configuration,
- this.volumeName, this.userName, this.pipelines, blockSize,
- volumeSize, flusher);
- cache.setCblockTargetMetrics(this.metrics);
- cache.setClientManager(this.clientManager);
-
- // TODO : Support user configurable maximum size.
- long cacheSize = computeCacheSize(this.configuration, this.volumeSize,
- this.blockSize);
- cache.setCurrentCacheSize(cacheSize);
- return cache;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
deleted file mode 100644
index 26c174f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import java.nio.ByteBuffer;
-
-/**
- * Impl class for LogicalBlock.
- */
-public class DiskBlock implements LogicalBlock {
- private ByteBuffer data;
- private long blockID;
- private boolean persisted;
-
- /**
- * Constructs a DiskBlock Class from the following params.
- * @param blockID - 64-bit block ID
- * @param data - Byte Array
- * @param persisted - Flag which tells us if this is persisted to remote
- */
- public DiskBlock(long blockID, byte[] data, boolean persisted) {
- if (data !=null) {
- this.data = ByteBuffer.wrap(data);
- }
- this.blockID = blockID;
- this.persisted = persisted;
- }
-
- @Override
- public ByteBuffer getData() {
- return data;
- }
-
- /**
- * Frees the byte buffer since we don't need it any more.
- */
- @Override
- public void clearData() {
- data.clear();
- }
-
- @Override
- public long getBlockID() {
- return blockID;
- }
-
- @Override
- public boolean isPersisted() {
- return persisted;
- }
-
- /**
- * Sets the value of persisted.
- * @param value - True if this has been persisted to container, false
- * otherwise.
- */
- public void setPersisted(boolean value) {
- persisted = value;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
deleted file mode 100644
index 533e919..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
+++ /dev/null
@@ -1,263 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
-
-import com.google.common.primitives.Longs;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Reads blocks from the container via the local cache.
- */
-public class SyncBlockReader {
- private static final Logger LOG =
- LoggerFactory.getLogger(SyncBlockReader.class);
-
- /**
- * Update Queue - The reason why we have the queue is that we want to
- * return the block as soon as we read it from the containers. This queue
- * is work queue which will take the read block and update the cache.
- * During testing we found levelDB is slow during writes, hence we wanted
- * to return as block as soon as possible and update levelDB asynchronously.
- */
- private final static int QUEUE_SIZE = 1024;
- /**
- * Config.
- */
- private final Configuration conf;
- /**
- * The parent cache this reader is operating against.
- */
- private final CBlockLocalCache parentCache;
- private final BlockingQueue<Runnable> updateQueue;
-
- /**
- * executor is used for running LevelDB updates. In future, we might do
- * read-aheads and this pool is useful for that too. The reason why we
- * don't share an executor for reads and writes is because the write task
- * is couple of magnitude slower than read task. So we don't want the
- * update DB to queue up behind the writes.
- */
- private final ThreadPoolExecutor executor;
-
- /**
- * Number of threads that pool starts with.
- */
- private final int corePoolSize = 1;
- /**
- * Maximum number of threads our pool will ever create.
- */
- private final int maxPoolSize = 10;
- /**
- * The idle time a thread hangs around waiting for work. if we don't find
- * new work in 60 seconds the worker thread is killed.
- */
- private final long keepAlive = 60L;
-
- /**
- * Constructs a SyncBlock reader.
- *
- * @param conf - Configuration
- * @param cache - Cache
- */
- public SyncBlockReader(Configuration conf, CBlockLocalCache cache) {
- this.conf = conf;
- this.parentCache = cache;
- updateQueue = new ArrayBlockingQueue<>(QUEUE_SIZE, true);
- ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
- .setNameFormat("SyncBlockReader Thread #%d").setDaemon(true).build();
- executor = new HadoopThreadPoolExecutor(
- corePoolSize, maxPoolSize, keepAlive, TimeUnit.SECONDS,
- updateQueue, workerThreadFactory,
- new ThreadPoolExecutor.CallerRunsPolicy());
- }
-
- /**
- * Returns the cache DB.
- *
- * @return LevelDB
- */
- private LevelDBStore getCacheDB() {
- return parentCache.getCacheDB();
- }
-
- /**
- * Returns data from the local cache if found, else reads from the remote
- * container.
- *
- * @param blockID - blockID
- * @return LogicalBlock
- */
- LogicalBlock readBlock(long blockID) throws IOException {
- XceiverClientSpi client = null;
- byte[] data = getblockFromDB(blockID);
- if (data != null) {
- parentCache.getTargetMetrics().incNumReadCacheHits();
- return new DiskBlock(blockID, data, false);
- }
-
- parentCache.getTargetMetrics().incNumReadCacheMiss();
- try {
- client = parentCache.getClientManager()
- .acquireClient(parentCache.getPipeline(blockID));
- LogicalBlock block = getBlockFromContainer(blockID, client);
- return block;
- } catch (Exception ex) {
- parentCache.getTargetMetrics().incNumFailedReadBlocks();
- LOG.error("read failed for BlockId: {}", blockID, ex);
- throw ex;
- } finally {
- if (client != null) {
- parentCache.getClientManager().releaseClient(client);
- }
- }
- }
-
- /**
- * Gets data from the DB if it exists.
- *
- * @param blockID - block id
- * @return data
- */
- private byte[] getblockFromDB(long blockID) {
- try {
- if(parentCache.isShortCircuitIOEnabled()) {
- long startTime = Time.monotonicNow();
- byte[] data = getCacheDB().get(Longs.toByteArray(blockID));
- long endTime = Time.monotonicNow();
-
- if (parentCache.isTraceEnabled()) {
- parentCache.getTracer().info(
- "Task=ReadTaskDBRead,BlockID={},SHA={},Time={}",
- blockID, (data != null && data.length > 0)
- ? DigestUtils.sha256Hex(data) : null,
- endTime - startTime);
- }
- parentCache.getTargetMetrics().updateDBReadLatency(
- endTime - startTime);
- return data;
- }
-
-
- } catch (DBException dbe) {
- LOG.error("Error while reading from cacheDB.", dbe);
- throw dbe;
- }
- return null;
- }
-
-
- /**
- * Returns a block from a Remote Container. if the key is not found on a
- * remote container we just return a block initialzied with zeros.
- *
- * @param blockID - blockID
- * @param client - client
- * @return LogicalBlock
- * @throws IOException
- */
- private LogicalBlock getBlockFromContainer(long blockID,
- XceiverClientSpi client) throws IOException {
- String containerName = parentCache.getPipeline(blockID).getContainerName();
- try {
- long startTime = Time.monotonicNow();
- ContainerProtos.GetSmallFileResponseProto response =
- ContainerProtocolCalls.readSmallFile(client, containerName,
- Long.toString(blockID), parentCache.getTraceID(blockID));
- long endTime = Time.monotonicNow();
- if (parentCache.isTraceEnabled()) {
- parentCache.getTracer().info(
- "Task=ReadTaskContainerRead,BlockID={},SHA={},Time={}",
- blockID, response.getData().getData().toByteArray().length > 0 ?
- DigestUtils.sha256Hex(response.getData()
- .getData().toByteArray()) : null, endTime - startTime);
- }
-
- parentCache.getTargetMetrics().updateContainerReadLatency(
- endTime - startTime);
- DiskBlock block = new DiskBlock(blockID,
- response.getData().getData().toByteArray(), false);
-
- if(parentCache.isShortCircuitIOEnabled()) {
- queueUpdateTask(block);
- }
-
- return block;
- } catch (IOException ex) {
- if (ex instanceof StorageContainerException) {
- parentCache.getTargetMetrics().incNumReadLostBlocks();
- StorageContainerException sce = (StorageContainerException) ex;
- if (sce.getResult() == ContainerProtos.Result.NO_SUCH_KEY ||
- sce.getResult() == ContainerProtos.Result.IO_EXCEPTION) {
- return new DiskBlock(blockID, new byte[parentCache.getBlockSize()],
- false);
- }
- }
- throw ex;
- }
- }
-
- /**
- * Updates the cache with the block that we just read.
- *
- * @param block
- */
- private void queueUpdateTask(final DiskBlock block) {
- Runnable updateTask = () -> {
- if(block.getData().array().length > 0) {
- getCacheDB().put(Longs.toByteArray(block.getBlockID()),
- block.getData().array());
- block.setPersisted(true);
- } else {
- LOG.error("Refusing to update the a null block in the local cache.");
- }
- };
- if (this.executor.isShutdown() || this.executor.isTerminated()) {
- LOG.error("Thread executor is not running.");
- } else {
- this.executor.submit(updateTask);
- }
- }
-
- /**
- * This is a read operation, we don't care if we updated the cache with the
- * last block e read.
- */
- void shutdown() {
- this.executor.shutdownNow();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
deleted file mode 100644
index dfac110..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
deleted file mode 100644
index 47f76b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper.cache;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
deleted file mode 100644
index 62fcd66..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerManagerImpl.java
+++ /dev/null
@@ -1,1101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.*;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.util.ReflectionUtils;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMStorageReport;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.utils.MetadataKeyFilters;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.security.DigestInputStream;
-import java.security.DigestOutputStream;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentNavigableMap;
-import java.util.concurrent.ConcurrentSkipListMap;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_EXISTS;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_NOT_FOUND;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.INVALID_CONFIG;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.IO_EXCEPTION;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.NO_SUCH_ALGORITHM;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNABLE_TO_READ_METADATA_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNSUPPORTED_REQUEST;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.ERROR_IN_COMPACT_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNCLOSED_CONTAINER_IO;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
-
-/**
- * A Generic ContainerManagerImpl that will be called from Ozone
- * ContainerManagerImpl. This allows us to support delta changes to ozone
- * version without having to rewrite the containerManager.
- */
-public class ContainerManagerImpl implements ContainerManager {
- static final Logger LOG =
- LoggerFactory.getLogger(ContainerManagerImpl.class);
-
- private final ConcurrentSkipListMap<String, ContainerStatus>
- containerMap = new ConcurrentSkipListMap<>();
-
- // Use a non-fair RW lock for better throughput, we may revisit this decision
- // if this causes fairness issues.
- private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
- private ContainerLocationManager locationManager;
- private ChunkManager chunkManager;
- private KeyManager keyManager;
- private Configuration conf;
- private DatanodeID datanodeID;
-
- private ContainerDeletionChoosingPolicy containerDeletionChooser;
- private ContainerReportManager containerReportManager;
-
- /**
- * Init call that sets up a container Manager.
- *
- * @param config - Configuration.
- * @param containerDirs - List of Metadata Container locations.
- * @param datanode - Datanode ID.
- * @throws IOException
- */
- @Override
- public void init(
- Configuration config, List<StorageLocation> containerDirs,
- DatanodeID datanode) throws IOException {
- Preconditions.checkNotNull(config, "Config must not be null");
- Preconditions.checkNotNull(containerDirs, "Container directories cannot " +
- "be null");
- Preconditions.checkNotNull(datanode, "Datanode ID cannot " +
- "be null");
-
- Preconditions.checkState(containerDirs.size() > 0, "Number of container" +
- " directories must be greater than zero.");
-
- this.conf = config;
- this.datanodeID = datanode;
-
- readLock();
- try {
- containerDeletionChooser = ReflectionUtils.newInstance(conf.getClass(
- ScmConfigKeys.OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY,
- TopNOrderedContainerDeletionChoosingPolicy.class,
- ContainerDeletionChoosingPolicy.class), conf);
-
- for (StorageLocation path : containerDirs) {
- File directory = Paths.get(path.getNormalizedUri()).toFile();
- if (!directory.exists() && !directory.mkdirs()) {
- LOG.error("Container metadata directory doesn't exist "
- + "and cannot be created. Path: {}", path.toString());
- throw new StorageContainerException("Container metadata "
- + "directory doesn't exist and cannot be created " + path
- .toString(), INVALID_CONFIG);
- }
-
- // TODO: This will fail if any directory is invalid.
- // We should fix this to handle invalid directories and continue.
- // Leaving it this way to fail fast for time being.
- if (!directory.isDirectory()) {
- LOG.error("Invalid path to container metadata directory. path: {}",
- path.toString());
- throw new StorageContainerException("Invalid path to container " +
- "metadata directory." + path, INVALID_CONFIG);
- }
- LOG.info("Loading containers under {}", path);
- File[] files = directory.listFiles(new ContainerFilter());
- if (files != null) {
- for (File containerFile : files) {
- LOG.debug("Loading container {}", containerFile);
- String containerPath =
- ContainerUtils.getContainerNameFromFile(containerFile);
- Preconditions.checkNotNull(containerPath, "Container path cannot" +
- " be null");
- readContainerInfo(containerPath);
- }
- }
- }
-
- List<StorageLocation> dataDirs = new LinkedList<>();
- for (String dir : config.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
- StorageLocation location = StorageLocation.parse(dir);
- dataDirs.add(location);
- }
- this.locationManager =
- new ContainerLocationManagerImpl(containerDirs, dataDirs, config);
-
- this.containerReportManager =
- new ContainerReportManagerImpl(config);
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Reads the Container Info from a file and verifies that checksum match. If
- * the checksums match, then that file is added to containerMap.
- *
- * @param containerName - Name which points to the persisted container.
- * @throws StorageContainerException
- */
- private void readContainerInfo(String containerName)
- throws StorageContainerException {
- Preconditions.checkState(containerName.length() > 0,
- "Container name length cannot be zero.");
- FileInputStream containerStream = null;
- DigestInputStream dis = null;
- FileInputStream metaStream = null;
- Path cPath = Paths.get(containerName).getFileName();
- String keyName = null;
- if (cPath != null) {
- keyName = cPath.toString();
- }
- Preconditions.checkNotNull(keyName,
- "Container Name to container key mapping is null");
-
- try {
- String containerFileName = containerName.concat(CONTAINER_EXTENSION);
- String metaFileName = containerName.concat(CONTAINER_META);
-
- containerStream = new FileInputStream(containerFileName);
-
- metaStream = new FileInputStream(metaFileName);
-
- MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-
- dis = new DigestInputStream(containerStream, sha);
-
- ContainerProtos.ContainerData containerDataProto =
- ContainerProtos.ContainerData.parseDelimitedFrom(dis);
- ContainerData containerData;
- if (containerDataProto == null) {
- // Sometimes container metadata might have been created but empty,
- // when loading the info we get a null, this often means last time
- // SCM was ending up at some middle phase causing that the metadata
- // was not populated. Such containers are marked as inactive.
- containerMap.put(keyName, new ContainerStatus(null));
- return;
- }
- containerData = ContainerData.getFromProtBuf(containerDataProto, conf);
- ContainerProtos.ContainerMeta meta =
- ContainerProtos.ContainerMeta.parseDelimitedFrom(metaStream);
- if (meta != null && !DigestUtils.sha256Hex(sha.digest())
- .equals(meta.getHash())) {
- // This means we were not able read data from the disk when booted the
- // datanode. We are going to rely on SCM understanding that we don't
- // have valid data for this container when we send container reports.
- // Hopefully SCM will ask us to delete this container and rebuild it.
- LOG.error("Invalid SHA found for container data. Name :{}"
- + "cowardly refusing to read invalid data", containerName);
- containerMap.put(keyName, new ContainerStatus(null));
- return;
- }
-
- ContainerStatus containerStatus = new ContainerStatus(containerData);
- // Initialize pending deletion blocks count in in-memory
- // container status.
- MetadataStore metadata = KeyUtils.getDB(containerData, conf);
- List<Map.Entry<byte[], byte[]>> underDeletionBlocks = metadata
- .getSequentialRangeKVs(null, Integer.MAX_VALUE,
- MetadataKeyFilters.getDeletingKeyFilter());
- containerStatus.incrPendingDeletionBlocks(underDeletionBlocks.size());
-
- List<Map.Entry<byte[], byte[]>> liveKeys = metadata
- .getRangeKVs(null, Integer.MAX_VALUE,
- MetadataKeyFilters.getNormalKeyFilter());
-
- // Get container bytesUsed upon loading container
- // The in-memory state is updated upon key write or delete
- // TODO: update containerDataProto and persist it into container MetaFile
- long bytesUsed = 0;
- bytesUsed = liveKeys.parallelStream().mapToLong(e-> {
- KeyData keyData;
- try {
- keyData = KeyUtils.getKeyData(e.getValue());
- return keyData.getSize();
- } catch (IOException ex) {
- return 0L;
- }
- }).sum();
- containerStatus.setBytesUsed(bytesUsed);
-
- containerMap.put(keyName, containerStatus);
- } catch (IOException | NoSuchAlgorithmException ex) {
- LOG.error("read failed for file: {} ex: {}", containerName,
- ex.getMessage());
-
- // TODO : Add this file to a recovery Queue.
-
- // Remember that this container is busted and we cannot use it.
- containerMap.put(keyName, new ContainerStatus(null));
- throw new StorageContainerException("Unable to read container info",
- UNABLE_TO_READ_METADATA_DB);
- } finally {
- IOUtils.closeStream(dis);
- IOUtils.closeStream(containerStream);
- IOUtils.closeStream(metaStream);
- }
- }
-
- /**
- * Creates a container with the given name.
- *
- * @param pipeline -- Nodes which make up this container.
- * @param containerData - Container Name and metadata.
- * @throws StorageContainerException - Exception
- */
- @Override
- public void createContainer(Pipeline pipeline, ContainerData containerData)
- throws StorageContainerException {
- Preconditions.checkNotNull(containerData, "Container data cannot be null");
- writeLock();
- try {
- if (containerMap.containsKey(containerData.getName())) {
- LOG.debug("container already exists. {}", containerData.getName());
- throw new StorageContainerException("container already exists.",
- CONTAINER_EXISTS);
- }
-
- // This is by design. We first write and close the
- // container Info and metadata to a directory.
- // Then read back and put that info into the containerMap.
- // This allows us to make sure that our write is consistent.
-
- writeContainerInfo(containerData, false);
- File cFile = new File(containerData.getContainerPath());
- readContainerInfo(ContainerUtils.getContainerNameFromFile(cFile));
- } catch (NoSuchAlgorithmException ex) {
- LOG.error("Internal error: We seem to be running a JVM without a " +
- "needed hash algorithm.");
- throw new StorageContainerException("failed to create container",
- NO_SUCH_ALGORITHM);
- } finally {
- writeUnlock();
- }
-
- }
-
- /**
- * Writes a container to a chosen location and updates the container Map.
- *
- * The file formats of ContainerData and Container Meta is the following.
- *
- * message ContainerData {
- * required string name = 1;
- * repeated KeyValue metadata = 2;
- * optional string dbPath = 3;
- * optional string containerPath = 4;
- * optional int64 bytesUsed = 5;
- * optional int64 size = 6;
- * }
- *
- * message ContainerMeta {
- * required string fileName = 1;
- * required string hash = 2;
- * }
- *
- * @param containerData - container Data
- * @param overwrite - Whether we are overwriting.
- * @throws StorageContainerException, NoSuchAlgorithmException
- */
- private void writeContainerInfo(ContainerData containerData,
- boolean overwrite)
- throws StorageContainerException, NoSuchAlgorithmException {
-
- Preconditions.checkNotNull(this.locationManager,
- "Internal error: location manager cannot be null");
-
- FileOutputStream containerStream = null;
- DigestOutputStream dos = null;
- FileOutputStream metaStream = null;
-
- try {
- Path metadataPath = null;
- Path location = (!overwrite) ? locationManager.getContainerPath():
- Paths.get(containerData.getContainerPath()).getParent();
- if (location == null) {
- throw new StorageContainerException(
- "Failed to get container file path.",
- CONTAINER_INTERNAL_ERROR);
- }
-
- File containerFile = ContainerUtils.getContainerFile(containerData,
- location);
- File metadataFile = ContainerUtils.getMetadataFile(containerData,
- location);
- String containerName = containerData.getContainerName();
-
- if(!overwrite) {
- ContainerUtils.verifyIsNewContainer(containerFile, metadataFile);
- metadataPath = this.locationManager.getDataPath(containerName);
- metadataPath = ContainerUtils.createMetadata(metadataPath,
- containerName, conf);
- } else {
- metadataPath = ContainerUtils.getMetadataDirectory(containerData);
- }
-
- containerStream = new FileOutputStream(containerFile);
- metaStream = new FileOutputStream(metadataFile);
- MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
-
- dos = new DigestOutputStream(containerStream, sha);
- containerData.setDBPath(metadataPath.resolve(
- ContainerUtils.getContainerDbFileName(containerName))
- .toString());
- containerData.setContainerPath(containerFile.toString());
-
- ContainerProtos.ContainerData protoData = containerData
- .getProtoBufMessage();
- protoData.writeDelimitedTo(dos);
-
- ContainerProtos.ContainerMeta protoMeta = ContainerProtos
- .ContainerMeta.newBuilder()
- .setFileName(containerFile.toString())
- .setHash(DigestUtils.sha256Hex(sha.digest()))
- .build();
- protoMeta.writeDelimitedTo(metaStream);
-
- } catch (IOException ex) {
- // TODO : we need to clean up partially constructed files
- // The proper way to do would be for a thread
- // to read all these 3 artifacts and make sure they are
- // sane. That info needs to come from the replication
- // pipeline, and if not consistent delete these file.
-
- // In case of ozone this is *not* a deal breaker since
- // SCM is guaranteed to generate unique container names.
- // The saving grace is that we check if we have residue files
- // lying around when creating a new container. We need to queue
- // this information to a cleaner thread.
-
- LOG.error("Creation of container failed. Name: {}, we might need to " +
- "cleanup partially created artifacts. ",
- containerData.getContainerName(), ex);
- throw new StorageContainerException("Container creation failed. ",
- ex, CONTAINER_INTERNAL_ERROR);
- } finally {
- IOUtils.closeStream(dos);
- IOUtils.closeStream(containerStream);
- IOUtils.closeStream(metaStream);
- }
- }
-
- /**
- * Deletes an existing container.
- *
- * @param pipeline - nodes that make this container.
- * @param containerName - name of the container.
- * @param forceDelete - whether this container should be deleted forcibly.
- * @throws StorageContainerException
- */
- @Override
- public void deleteContainer(Pipeline pipeline, String containerName,
- boolean forceDelete) throws StorageContainerException {
- Preconditions.checkNotNull(containerName, "Container name cannot be null");
- Preconditions.checkState(containerName.length() > 0,
- "Container name length cannot be zero.");
- writeLock();
- try {
- if (isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException(
- "Deleting an open container is not allowed.",
- UNCLOSED_CONTAINER_IO);
- }
-
- ContainerStatus status = containerMap.get(containerName);
- if (status == null) {
- LOG.debug("No such container. Name: {}", containerName);
- throw new StorageContainerException("No such container. Name : " +
- containerName, CONTAINER_NOT_FOUND);
- }
- if (status.getContainer() == null) {
- LOG.debug("Invalid container data. Name: {}", containerName);
- throw new StorageContainerException("Invalid container data. Name : " +
- containerName, CONTAINER_NOT_FOUND);
- }
- ContainerUtils.removeContainer(status.getContainer(), conf, forceDelete);
- containerMap.remove(containerName);
- } catch (StorageContainerException e) {
- throw e;
- } catch (IOException e) {
- // TODO : An I/O error during delete can leave partial artifacts on the
- // disk. We will need the cleaner thread to cleanup this information.
- LOG.error("Failed to cleanup container. Name: {}", containerName, e);
- throw new StorageContainerException(containerName, e, IO_EXCEPTION);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * A simple interface for container Iterations.
- * <p/>
- * This call make no guarantees about consistency of the data between
- * different list calls. It just returns the best known data at that point of
- * time. It is possible that using this iteration you can miss certain
- * container from the listing.
- *
- * @param prefix - Return keys that match this prefix.
- * @param count - how many to return
- * @param prevKey - Previous Key Value or empty String.
- * @param data - Actual containerData
- * @throws StorageContainerException
- */
- @Override
- public void listContainer(String prefix, long count, String prevKey,
- List<ContainerData> data) throws StorageContainerException {
- // TODO : Support list with Prefix and PrevKey
- Preconditions.checkNotNull(data,
- "Internal assertion: data cannot be null");
- readLock();
- try {
- ConcurrentNavigableMap<String, ContainerStatus> map;
- if (prevKey == null || prevKey.isEmpty()) {
- map = containerMap.tailMap(containerMap.firstKey(), true);
- } else {
- map = containerMap.tailMap(prevKey, false);
- }
-
- int currentCount = 0;
- for (ContainerStatus entry : map.values()) {
- if (currentCount < count) {
- data.add(entry.getContainer());
- currentCount++;
- } else {
- return;
- }
- }
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Get metadata about a specific container.
- *
- * @param containerName - Name of the container
- * @return ContainerData - Container Data.
- * @throws StorageContainerException
- */
- @Override
- public ContainerData readContainer(String containerName) throws
- StorageContainerException {
- Preconditions.checkNotNull(containerName, "Container name cannot be null");
- Preconditions.checkState(containerName.length() > 0,
- "Container name length cannot be zero.");
- if (!containerMap.containsKey(containerName)) {
- throw new StorageContainerException("Unable to find the container. Name: "
- + containerName, CONTAINER_NOT_FOUND);
- }
- ContainerData cData = containerMap.get(containerName).getContainer();
- if (cData == null) {
- throw new StorageContainerException("Invalid container data. Name: "
- + containerName, CONTAINER_INTERNAL_ERROR);
- }
- return cData;
- }
-
- /**
- * Closes a open container, if it is already closed or does not exist a
- * StorageContainerException is thrown.
- *
- * @param containerName - Name of the container.
- * @throws StorageContainerException
- */
- @Override
- public void closeContainer(String containerName)
- throws StorageContainerException, NoSuchAlgorithmException {
- ContainerData containerData = readContainer(containerName);
- containerData.closeContainer();
- writeContainerInfo(containerData, true);
- MetadataStore db = KeyUtils.getDB(containerData, conf);
-
- // It is ok if this operation takes a bit of time.
- // Close container is not expected to be instantaneous.
- try {
- db.compactDB();
- } catch (IOException e) {
- LOG.error("Error in DB compaction while closing container", e);
- throw new StorageContainerException(e, ERROR_IN_COMPACT_DB);
- }
-
- // Active is different from closed. Closed means it is immutable, active
- // false means we have some internal error that is happening to this
- // container. This is a way to track damaged containers if we have an
- // I/O failure, this allows us to take quick action in case of container
- // issues.
-
- ContainerStatus status = new ContainerStatus(containerData);
- containerMap.put(containerName, status);
- }
-
- @Override
- public void updateContainer(Pipeline pipeline, String containerName,
- ContainerData data, boolean forceUpdate)
- throws StorageContainerException {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- Preconditions.checkNotNull(containerName, "Container name cannot be null");
- Preconditions.checkNotNull(data, "Container data cannot be null");
- FileOutputStream containerStream = null;
- DigestOutputStream dos = null;
- MessageDigest sha = null;
- File containerFileBK = null, containerFile = null;
- boolean deleted = false;
-
- if(!containerMap.containsKey(containerName)) {
- throw new StorageContainerException("Container doesn't exist. Name :"
- + containerName, CONTAINER_NOT_FOUND);
- }
-
- try {
- sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
- } catch (NoSuchAlgorithmException e) {
- throw new StorageContainerException("Unable to create Message Digest,"
- + " usually this is a java configuration issue.",
- NO_SUCH_ALGORITHM);
- }
-
- try {
- Path location = locationManager.getContainerPath();
- ContainerData orgData = containerMap.get(containerName).getContainer();
- if (orgData == null) {
- // updating a invalid container
- throw new StorageContainerException("Update a container with invalid" +
- "container meta data", CONTAINER_INTERNAL_ERROR);
- }
-
- if (!forceUpdate && !orgData.isOpen()) {
- throw new StorageContainerException(
- "Update a closed container is not allowed. Name: " + containerName,
- UNSUPPORTED_REQUEST);
- }
-
- containerFile = ContainerUtils.getContainerFile(orgData, location);
- // If forceUpdate is true, there is no need to check
- // whether the container file exists.
- if (!forceUpdate) {
- if (!containerFile.exists() || !containerFile.canWrite()) {
- throw new StorageContainerException(
- "Container file not exists or corrupted. Name: " + containerName,
- CONTAINER_INTERNAL_ERROR);
- }
-
- // Backup the container file
- containerFileBK = File.createTempFile(
- "tmp_" + System.currentTimeMillis() + "_",
- containerFile.getName(), containerFile.getParentFile());
- FileUtils.copyFile(containerFile, containerFileBK);
-
- deleted = containerFile.delete();
- containerStream = new FileOutputStream(containerFile);
- dos = new DigestOutputStream(containerStream, sha);
-
- ContainerProtos.ContainerData protoData = data.getProtoBufMessage();
- protoData.writeDelimitedTo(dos);
- }
-
- // Update the in-memory map
- ContainerStatus newStatus = new ContainerStatus(data);
- containerMap.replace(containerName, newStatus);
- } catch (IOException e) {
- // Restore the container file from backup
- if(containerFileBK != null && containerFileBK.exists() && deleted) {
- if(containerFile.delete()
- && containerFileBK.renameTo(containerFile)) {
- throw new StorageContainerException("Container update failed,"
- + " container data restored from the backup.",
- CONTAINER_INTERNAL_ERROR);
- } else {
- throw new StorageContainerException(
- "Failed to restore container data from the backup. Name: "
- + containerName, CONTAINER_INTERNAL_ERROR);
- }
- } else {
- throw new StorageContainerException(
- e.getMessage(), CONTAINER_INTERNAL_ERROR);
- }
- } finally {
- if (containerFileBK != null && containerFileBK.exists()) {
- if(!containerFileBK.delete()) {
- LOG.warn("Unable to delete container file backup : {}.",
- containerFileBK.getAbsolutePath());
- }
- }
- IOUtils.closeStream(dos);
- IOUtils.closeStream(containerStream);
- }
- }
-
- @VisibleForTesting
- protected File getContainerFile(ContainerData data) throws IOException {
- return ContainerUtils.getContainerFile(data,
- this.locationManager.getContainerPath());
- }
-
- /**
- * Checks if a container exists.
- *
- * @param containerName - Name of the container.
- * @return true if the container is open false otherwise.
- * @throws StorageContainerException - Throws Exception if we are not able to
- * find the container.
- */
- @Override
- public boolean isOpen(String containerName) throws StorageContainerException {
- final ContainerStatus status = containerMap.get(containerName);
- if (status == null) {
- throw new StorageContainerException(
- "Container status not found: " + containerName, CONTAINER_NOT_FOUND);
- }
- final ContainerData cData = status.getContainer();
- if (cData == null) {
- throw new StorageContainerException(
- "Container not found: " + containerName, CONTAINER_NOT_FOUND);
- }
- return cData.isOpen();
- }
-
- /**
- * Supports clean shutdown of container.
- *
- * @throws IOException
- */
- @Override
- public void shutdown() throws IOException {
- Preconditions.checkState(this.hasWriteLock(),
- "Assumption that we are holding the lock violated.");
- this.containerMap.clear();
- this.locationManager.shutdown();
- }
-
-
- @VisibleForTesting
- public ConcurrentSkipListMap<String, ContainerStatus> getContainerMap() {
- return containerMap;
- }
-
- /**
- * Acquire read lock.
- */
- @Override
- public void readLock() {
- this.lock.readLock().lock();
-
- }
-
- @Override
- public void readLockInterruptibly() throws InterruptedException {
- this.lock.readLock().lockInterruptibly();
- }
-
- /**
- * Release read lock.
- */
- @Override
- public void readUnlock() {
- this.lock.readLock().unlock();
- }
-
- /**
- * Check if the current thread holds read lock.
- */
- @Override
- public boolean hasReadLock() {
- return this.lock.readLock().tryLock();
- }
-
- /**
- * Acquire write lock.
- */
- @Override
- public void writeLock() {
- this.lock.writeLock().lock();
- }
-
- /**
- * Acquire write lock, unless interrupted while waiting.
- */
- @Override
- public void writeLockInterruptibly() throws InterruptedException {
- this.lock.writeLock().lockInterruptibly();
-
- }
-
- /**
- * Release write lock.
- */
- @Override
- public void writeUnlock() {
- this.lock.writeLock().unlock();
-
- }
-
- /**
- * Check if the current thread holds write lock.
- */
- @Override
- public boolean hasWriteLock() {
- return this.lock.writeLock().isHeldByCurrentThread();
- }
-
- public ChunkManager getChunkManager() {
- return this.chunkManager;
- }
-
- /**
- * Sets the chunk Manager.
- *
- * @param chunkManager - Chunk Manager
- */
- public void setChunkManager(ChunkManager chunkManager) {
- this.chunkManager = chunkManager;
- }
-
- /**
- * Gets the Key Manager.
- *
- * @return KeyManager.
- */
- @Override
- public KeyManager getKeyManager() {
- return this.keyManager;
- }
-
- /**
- * Get the node report.
- * @return node report.
- */
- @Override
- public SCMNodeReport getNodeReport() throws IOException {
- StorageLocationReport[] reports = locationManager.getLocationReport();
- SCMNodeReport.Builder nrb = SCMNodeReport.newBuilder();
- for (int i = 0; i < reports.length; i++) {
- SCMStorageReport.Builder srb = SCMStorageReport.newBuilder();
- nrb.addStorageReport(i, srb.setStorageUuid(reports[i].getId())
- .setCapacity(reports[i].getCapacity())
- .setScmUsed(reports[i].getScmUsed())
- .setRemaining(reports[i].getRemaining())
- .build());
- }
- return nrb.build();
- }
-
-
- /**
- * Gets container reports.
- *
- * @return List of all closed containers.
- * @throws IOException
- */
- @Override
- public List<ContainerData> getContainerReports() throws IOException {
- LOG.debug("Starting container report iteration.");
- // No need for locking since containerMap is a ConcurrentSkipListMap
- // And we can never get the exact state since close might happen
- // after we iterate a point.
- return containerMap.entrySet().stream()
- .filter(containerStatus ->
- !containerStatus.getValue().getContainer().isOpen())
- .map(containerStatus -> containerStatus.getValue().getContainer())
- .collect(Collectors.toList());
- }
-
- /**
- * Get container report.
- *
- * @return The container report.
- * @throws IOException
- */
- @Override
- public ContainerReportsRequestProto getContainerReport() throws IOException {
- LOG.debug("Starting container report iteration.");
- // No need for locking since containerMap is a ConcurrentSkipListMap
- // And we can never get the exact state since close might happen
- // after we iterate a point.
- List<ContainerStatus> containers = containerMap.values().stream()
- .collect(Collectors.toList());
-
- ContainerReportsRequestProto.Builder crBuilder =
- ContainerReportsRequestProto.newBuilder();
-
- // TODO: support delta based container report
- crBuilder.setDatanodeID(datanodeID.getProtoBufMessage())
- .setType(ContainerReportsRequestProto.reportType.fullReport);
-
- for (ContainerStatus container: containers) {
- StorageContainerDatanodeProtocolProtos.ContainerInfo.Builder ciBuilder =
- StorageContainerDatanodeProtocolProtos.ContainerInfo.newBuilder();
- ciBuilder.setContainerName(container.getContainer().getContainerName())
- .setSize(container.getContainer().getMaxSize())
- .setUsed(container.getContainer().getBytesUsed())
- .setKeyCount(container.getContainer().getKeyCount())
- .setReadCount(container.getReadCount())
- .setWriteCount(container.getWriteCount())
- .setReadBytes(container.getReadBytes())
- .setWriteBytes(container.getWriteBytes())
- .setContainerID(container.getContainer().getContainerID());
-
- if (container.getContainer().getHash() != null) {
- ciBuilder.setFinalhash(container.getContainer().getHash());
- }
- crBuilder.addReports(ciBuilder.build());
- }
-
- return crBuilder.build();
- }
-
- /**
- * Sets the Key Manager.
- *
- * @param keyManager - Key Manager.
- */
- @Override
- public void setKeyManager(KeyManager keyManager) {
- this.keyManager = keyManager;
- }
-
- /**
- * Filter out only container files from the container metadata dir.
- */
- private static class ContainerFilter implements FilenameFilter {
- /**
- * Tests if a specified file should be included in a file list.
- *
- * @param dir the directory in which the file was found.
- * @param name the name of the file.
- * @return <code>true</code> if and only if the name should be included in
- * the file list; <code>false</code> otherwise.
- */
- @Override
- public boolean accept(File dir, String name) {
- return name.endsWith(CONTAINER_EXTENSION);
- }
- }
-
- @Override
- public List<ContainerData> chooseContainerForBlockDeletion(
- int count) throws StorageContainerException {
- readLock();
- try {
- return containerDeletionChooser.chooseContainerForBlockDeletion(
- count, containerMap);
- } finally {
- readUnlock();
- }
- }
-
- @VisibleForTesting
- public ContainerDeletionChoosingPolicy getContainerDeletionChooser() {
- return containerDeletionChooser;
- }
-
- @Override
- public void incrPendingDeletionBlocks(int numBlocks, String containerId) {
- writeLock();
- try {
- ContainerStatus status = containerMap.get(containerId);
- status.incrPendingDeletionBlocks(numBlocks);
- } finally {
- writeUnlock();
- }
- }
-
- @Override
- public void decrPendingDeletionBlocks(int numBlocks, String containerId) {
- writeLock();
- try {
- ContainerStatus status = containerMap.get(containerId);
- status.decrPendingDeletionBlocks(numBlocks);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Increase the read count of the container.
- *
- * @param containerName - Name of the container.
- */
- @Override
- public void incrReadCount(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- status.incrReadCount();
- }
-
- public long getReadCount(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- return status.getReadCount();
- }
-
- /**
- * Increse the read counter for bytes read from the container.
- *
- * @param containerName - Name of the container.
- * @param readBytes - bytes read from the container.
- */
- @Override
- public void incrReadBytes(String containerName, long readBytes) {
- ContainerStatus status = containerMap.get(containerName);
- status.incrReadBytes(readBytes);
- }
-
- public long getReadBytes(String containerName) {
- readLock();
- try {
- ContainerStatus status = containerMap.get(containerName);
- return status.getReadBytes();
- } finally {
- readUnlock();
- }
- }
-
- /**
- * Increase the write count of the container.
- *
- * @param containerName - Name of the container.
- */
- @Override
- public void incrWriteCount(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- status.incrWriteCount();
- }
-
- public long getWriteCount(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- return status.getWriteCount();
- }
-
- /**
- * Increse the write counter for bytes write into the container.
- *
- * @param containerName - Name of the container.
- * @param writeBytes - bytes write into the container.
- */
- @Override
- public void incrWriteBytes(String containerName, long writeBytes) {
- ContainerStatus status = containerMap.get(containerName);
- status.incrWriteBytes(writeBytes);
- }
-
- public long getWriteBytes(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- return status.getWriteBytes();
- }
-
- /**
- * Increase the bytes used by the container.
- *
- * @param containerName - Name of the container.
- * @param used - additional bytes used by the container.
- * @return the current bytes used.
- */
- @Override
- public long incrBytesUsed(String containerName, long used) {
- ContainerStatus status = containerMap.get(containerName);
- return status.incrBytesUsed(used);
- }
-
- /**
- * Decrease the bytes used by the container.
- *
- * @param containerName - Name of the container.
- * @param used - additional bytes reclaimed by the container.
- * @return the current bytes used.
- */
- @Override
- public long decrBytesUsed(String containerName, long used) {
- ContainerStatus status = containerMap.get(containerName);
- return status.decrBytesUsed(used);
- }
-
- public long getBytesUsed(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- return status.getBytesUsed();
- }
-
- /**
- * Get the number of keys in the container.
- *
- * @param containerName - Name of the container.
- * @return the current key count.
- */
- @Override
- public long getNumKeys(String containerName) {
- ContainerStatus status = containerMap.get(containerName);
- return status.getNumKeys(); }
-
- /**
- * Get the container report state to send via HB to SCM.
- *
- * @return container report state.
- */
- @Override
- public ReportState getContainerReportState() {
- return containerReportManager.getContainerReportState();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
deleted file mode 100644
index 4669737..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerReportManagerImpl.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerReportManager;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.util.Time;
-
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * Class wraps the container report operations on datanode.
- * // TODO: support incremental/delta container report
- */
-public class ContainerReportManagerImpl implements ContainerReportManager {
- // Last non-empty container report time
- private long lastContainerReportTime;
- private final long containerReportInterval;
- private final long heartbeatInterval;
- private AtomicLong reportCount;
- private static final ReportState NO_CONTAINER_REPORTSTATE =
- ReportState.newBuilder()
- .setState(ReportState.states.noContainerReports)
- .setCount(0).build();
-
- public ContainerReportManagerImpl(Configuration config) {
- this.lastContainerReportTime = -1;
- this.reportCount = new AtomicLong(0L);
- this.containerReportInterval = config.getTimeDuration(
- OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL,
- OzoneConfigKeys.OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.heartbeatInterval = OzoneClientUtils.getScmHeartbeatInterval(config);
- }
-
- public ReportState getContainerReportState() {
- if (lastContainerReportTime < 0) {
- return getFullContainerReportState();
- } else {
- // Add a random delay (0~30s) on top of the container report
- // interval (60s) so tha the SCM is overwhelmed by the container reports
- // sent in sync.
- if (Time.monotonicNow() - lastContainerReportTime >
- (containerReportInterval + getRandomReportDelay())) {
- return getFullContainerReportState();
- } else {
- return getNoContainerReportState();
- }
- }
- }
-
- private ReportState getFullContainerReportState() {
- ReportState.Builder rsBuilder = ReportState.newBuilder();
- rsBuilder.setState(ReportState.states.completeContinerReport);
- rsBuilder.setCount(reportCount.incrementAndGet());
- this.lastContainerReportTime = Time.monotonicNow();
- return rsBuilder.build();
- }
-
- private ReportState getNoContainerReportState() {
- return NO_CONTAINER_REPORTSTATE;
- }
-
- private long getRandomReportDelay() {
- return RandomUtils.nextLong(0, heartbeatInterval);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
deleted file mode 100644
index 5577323..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStatus.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class represents the state of a container. if the
- * container reading encountered an error when we boot up we will post that
- * info to a recovery queue and keep the info in the containerMap.
- * <p/>
- * if and when the issue is fixed, the expectation is that this entry will be
- * deleted by the recovery thread from the containerMap and will insert entry
- * instead of modifying this class.
- */
-public class ContainerStatus {
- private final ContainerData containerData;
-
- /**
- * Number of pending deletion blocks in container.
- */
- private int numPendingDeletionBlocks;
-
- private AtomicLong readBytes;
-
- private AtomicLong writeBytes;
-
- private AtomicLong readCount;
-
- private AtomicLong writeCount;
-
- /**
- * Creates a Container Status class.
- *
- * @param containerData - ContainerData.
- */
- ContainerStatus(ContainerData containerData) {
- this.numPendingDeletionBlocks = 0;
- this.containerData = containerData;
- this.readCount = new AtomicLong(0L);
- this.readBytes = new AtomicLong(0L);
- this.writeCount = new AtomicLong(0L);
- this.writeBytes = new AtomicLong(0L);
- }
-
- /**
- * Returns container if it is active. It is not active if we have had an
- * error and we are waiting for the background threads to fix the issue.
- *
- * @return ContainerData.
- */
- public ContainerData getContainer() {
- return containerData;
- }
-
- /**
- * Increase the count of pending deletion blocks.
- *
- * @param numBlocks increment number
- */
- public void incrPendingDeletionBlocks(int numBlocks) {
- this.numPendingDeletionBlocks += numBlocks;
- }
-
- /**
- * Decrease the count of pending deletion blocks.
- *
- * @param numBlocks decrement number
- */
- public void decrPendingDeletionBlocks(int numBlocks) {
- this.numPendingDeletionBlocks -= numBlocks;
- }
-
- /**
- * Get the number of pending deletion blocks.
- */
- public int getNumPendingDeletionBlocks() {
- return this.numPendingDeletionBlocks;
- }
-
- /**
- * Get the number of bytes read from the container.
- * @return the number of bytes read from the container.
- */
- public long getReadBytes() {
- return readBytes.get();
- }
-
- /**
- * Increase the number of bytes read from the container.
- * @param bytes number of bytes read.
- */
- public void incrReadBytes(long bytes) {
- this.readBytes.addAndGet(bytes);
- }
-
- /**
- * Get the number of times the container is read.
- * @return the number of times the container is read.
- */
- public long getReadCount() {
- return readCount.get();
- }
-
- /**
- * Increase the number of container read count by 1.
- */
- public void incrReadCount() {
- this.readCount.incrementAndGet();
- }
-
- /**
- * Get the number of bytes write into the container.
- * @return the number of bytes write into the container.
- */
- public long getWriteBytes() {
- return writeBytes.get();
- }
-
- /**
- * Increase the number of bytes write into the container.
- * @param bytes the number of bytes write into the container.
- */
- public void incrWriteBytes(long bytes) {
- this.writeBytes.addAndGet(bytes);
- }
-
- /**
- * Get the number of writes into the container.
- * @return the number of writes into the container.
- */
- public long getWriteCount() {
- return writeCount.get();
- }
-
- /**
- * Increase the number of writes into the container by 1.
- */
- public void incrWriteCount() {
- this.writeCount.incrementAndGet();
- }
-
- /**
- * Get the number of bytes used by the container.
- * @return the number of bytes used by the container.
- */
- public long getBytesUsed() {
- return containerData.getBytesUsed();
- }
-
- /**
- * Increase the number of bytes used by the container.
- * @param used number of bytes used by the container.
- * @return the current number of bytes used by the container afert increase.
- */
- public long incrBytesUsed(long used) {
- return containerData.addBytesUsed(used);
- }
-
- /**
- * Set the number of bytes used by the container.
- * @param used the number of bytes used by the container.
- */
- public void setBytesUsed(long used) {
- containerData.setBytesUsed(used);
- }
-
- /**
- * Decrease the number of bytes used by the container.
- * @param reclaimed the number of bytes reclaimed from the container.
- * @return the current number of bytes used by the container after decrease.
- */
- public long decrBytesUsed(long reclaimed) {
- return this.containerData.addBytesUsed(-1L * reclaimed);
- }
-
- /**
- * Get the maximum container size.
- * @return the maximum container size.
- */
- public long getMaxSize() {
- return containerData.getMaxSize();
- }
-
- /**
- * Set the maximum container size.
- * @param size the maximum container size.
- */
- public void setMaxSize(long size) {
- this.containerData.setMaxSize(size);
- }
-
- /**
- * Get the number of keys in the container.
- * @return the number of keys in the container.
- */
- public long getNumKeys() {
- return containerData.getKeyCount();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
deleted file mode 100644
index 07a3a53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerStorageLocation.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CachingGetSpaceUsed;
-import org.apache.hadoop.fs.DF;
-import org.apache.hadoop.fs.GetSpaceUsed;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.FileNotFoundException;
-import java.io.OutputStreamWriter;
-import java.io.FileOutputStream;
-import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
-import java.util.Scanner;
-
-import static org.apache.hadoop.util.RunJar.SHUTDOWN_HOOK_PRIORITY;
-
-/**
- * Class that wraps the space usage of the Datanode Container Storage Location
- * by SCM containers.
- */
-public class ContainerStorageLocation {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStorageLocation.class);
-
- private static final String DU_CACHE_FILE = "scmUsed";
- private volatile boolean scmUsedSaved = false;
-
- private final StorageLocation dataLocation;
- private final String storageUuId;
- private final DF usage;
- private final GetSpaceUsed scmUsage;
- private final File scmUsedFile;
-
- public ContainerStorageLocation(StorageLocation dataLoc, Configuration conf)
- throws IOException {
- this.dataLocation = dataLoc;
- this.storageUuId = DatanodeStorage.generateUuid();
- File dataDir = Paths.get(dataLoc.getNormalizedUri()).resolve(
- OzoneConsts.CONTAINER_PREFIX).toFile();
- // Initialize container data root if it does not exist as required by DF/DU
- if (!dataDir.exists()) {
- if (!dataDir.mkdirs()) {
- LOG.error("Unable to create the container storage location at : {}",
- dataDir);
- throw new IllegalArgumentException("Unable to create the container" +
- " storage location at : " + dataDir);
- }
- }
- scmUsedFile = new File(dataDir, DU_CACHE_FILE);
- // get overall disk usage
- this.usage = new DF(dataDir, conf);
- // get SCM specific usage
- this.scmUsage = new CachingGetSpaceUsed.Builder().setPath(dataDir)
- .setConf(conf)
- .setInitialUsed(loadScmUsed())
- .build();
-
- // Ensure scm usage is saved during shutdown.
- ShutdownHookManager.get().addShutdownHook(
- new Runnable() {
- @Override
- public void run() {
- if (!scmUsedSaved) {
- saveScmUsed();
- }
- }
- }, SHUTDOWN_HOOK_PRIORITY);
- }
-
- public URI getNormalizedUri() {
- return dataLocation.getNormalizedUri();
- }
-
- public String getStorageUuId() {
- return storageUuId;
- }
- public long getCapacity() {
- long capacity = usage.getCapacity();
- return (capacity > 0) ? capacity : 0;
- }
-
- public long getAvailable() throws IOException {
- long remaining = getCapacity() - getScmUsed();
- long available = usage.getAvailable();
- if (remaining > available) {
- remaining = available;
- }
- return (remaining > 0) ? remaining : 0;
- }
-
- public long getScmUsed() throws IOException{
- return scmUsage.getUsed();
- }
-
- public void shutdown() {
- saveScmUsed();
- scmUsedSaved = true;
-
- if (scmUsage instanceof CachingGetSpaceUsed) {
- IOUtils.cleanupWithLogger(null, ((CachingGetSpaceUsed) scmUsage));
- }
- }
-
- /**
- * Read in the cached DU value and return it if it is less than 600 seconds
- * old (DU update interval). Slight imprecision of scmUsed is not critical
- * and skipping DU can significantly shorten the startup time.
- * If the cached value is not available or too old, -1 is returned.
- */
- long loadScmUsed() {
- long cachedScmUsed;
- long mtime;
- Scanner sc;
-
- try {
- sc = new Scanner(scmUsedFile, "UTF-8");
- } catch (FileNotFoundException fnfe) {
- return -1;
- }
-
- try {
- // Get the recorded scmUsed from the file.
- if (sc.hasNextLong()) {
- cachedScmUsed = sc.nextLong();
- } else {
- return -1;
- }
- // Get the recorded mtime from the file.
- if (sc.hasNextLong()) {
- mtime = sc.nextLong();
- } else {
- return -1;
- }
-
- // Return the cached value if mtime is okay.
- if (mtime > 0 && (Time.now() - mtime < 600000L)) {
- LOG.info("Cached ScmUsed found for {} : {} ", dataLocation,
- cachedScmUsed);
- return cachedScmUsed;
- }
- return -1;
- } finally {
- sc.close();
- }
- }
-
- /**
- * Write the current scmUsed to the cache file.
- */
- void saveScmUsed() {
- if (scmUsedFile.exists() && !scmUsedFile.delete()) {
- LOG.warn("Failed to delete old scmUsed file in {}.", dataLocation);
- }
- OutputStreamWriter out = null;
- try {
- long used = getScmUsed();
- if (used > 0) {
- out = new OutputStreamWriter(new FileOutputStream(scmUsedFile),
- StandardCharsets.UTF_8);
- // mtime is written last, so that truncated writes won't be valid.
- out.write(Long.toString(used) + " " + Long.toString(Time.now()));
- out.flush();
- out.close();
- out = null;
- }
- } catch (IOException ioe) {
- // If write failed, the volume might be bad. Since the cache file is
- // not critical, log the error and continue.
- LOG.warn("Failed to write scmUsed to " + scmUsedFile, ioe);
- } finally {
- IOUtils.cleanupWithLogger(null, out);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
deleted file mode 100644
index 40bc7ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/Dispatcher.java
+++ /dev/null
@@ -1,708 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Type;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerMetrics;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerUtils;
-import org.apache.hadoop.ozone.container.common.helpers.FileUtils;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.LinkedList;
-import java.util.List;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.CLOSED_CONTAINER_IO;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.GET_SMALL_FILE_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.NO_SUCH_ALGORITHM;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Result.PUT_SMALL_FILE_ERROR;
-
-/**
- * Ozone Container dispatcher takes a call from the netty server and routes it
- * to the right handler function.
- */
-public class Dispatcher implements ContainerDispatcher {
- static final Logger LOG = LoggerFactory.getLogger(Dispatcher.class);
-
- private final ContainerManager containerManager;
- private ContainerMetrics metrics;
- private Configuration conf;
-
- /**
- * Constructs an OzoneContainer that receives calls from
- * XceiverServerHandler.
- *
- * @param containerManager - A class that manages containers.
- */
- public Dispatcher(ContainerManager containerManager, Configuration config) {
- Preconditions.checkNotNull(containerManager);
- this.containerManager = containerManager;
- this.metrics = null;
- this.conf = config;
- }
-
- @Override
- public void init() {
- this.metrics = ContainerMetrics.create(conf);
- }
-
- @Override
- public void shutdown() {
- }
-
- @Override
- public ContainerCommandResponseProto dispatch(
- ContainerCommandRequestProto msg) {
- LOG.trace("Command {}, trace ID: {} ", msg.getCmdType().toString(),
- msg.getTraceID());
- long startNanos = System.nanoTime();
- ContainerCommandResponseProto resp = null;
- try {
- Preconditions.checkNotNull(msg);
- Type cmdType = msg.getCmdType();
- metrics.incContainerOpcMetrics(cmdType);
- if ((cmdType == Type.CreateContainer) ||
- (cmdType == Type.DeleteContainer) ||
- (cmdType == Type.ReadContainer) ||
- (cmdType == Type.ListContainer) ||
- (cmdType == Type.UpdateContainer) ||
- (cmdType == Type.CloseContainer)) {
- resp = containerProcessHandler(msg);
- }
-
- if ((cmdType == Type.PutKey) ||
- (cmdType == Type.GetKey) ||
- (cmdType == Type.DeleteKey) ||
- (cmdType == Type.ListKey)) {
- resp = keyProcessHandler(msg);
- }
-
- if ((cmdType == Type.WriteChunk) ||
- (cmdType == Type.ReadChunk) ||
- (cmdType == Type.DeleteChunk)) {
- resp = chunkProcessHandler(msg);
- }
-
- if ((cmdType == Type.PutSmallFile) ||
- (cmdType == Type.GetSmallFile)) {
- resp = smallFileHandler(msg);
- }
-
- if (resp != null) {
- metrics.incContainerOpsLatencies(cmdType,
- System.nanoTime() - startNanos);
- return resp;
- }
-
- return ContainerUtils.unsupportedRequest(msg);
- } catch (StorageContainerException e) {
- // This useful since the trace ID will allow us to correlate failures.
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IllegalStateException | NullPointerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- }
- }
-
- public ContainerMetrics getContainerMetrics() {
- return metrics;
- }
-
- /**
- * Handles the all Container related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto containerProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
-
- switch (msg.getCmdType()) {
- case CreateContainer:
- return handleCreateContainer(msg);
-
- case DeleteContainer:
- return handleDeleteContainer(msg);
-
- case ListContainer:
- // TODO : Support List Container.
- return ContainerUtils.unsupportedRequest(msg);
-
- case UpdateContainer:
- return handleUpdateContainer(msg);
-
- case ReadContainer:
- return handleReadContainer(msg);
-
- case CloseContainer:
- return handleCloseContainer(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Handles the all key related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto keyProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
- switch (msg.getCmdType()) {
- case PutKey:
- return handlePutKey(msg);
-
- case GetKey:
- return handleGetKey(msg);
-
- case DeleteKey:
- return handleDeleteKey(msg);
-
- case ListKey:
- return ContainerUtils.unsupportedRequest(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
-
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Handles the all chunk related functionality.
- *
- * @param msg - command
- * @return - response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto chunkProcessHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- try {
- switch (msg.getCmdType()) {
- case WriteChunk:
- return handleWriteChunk(msg);
-
- case ReadChunk:
- return handleReadChunk(msg);
-
- case DeleteChunk:
- return handleDeleteChunk(msg);
-
- case ListChunk:
- return ContainerUtils.unsupportedRequest(msg);
-
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException ex) {
- LOG.warn("Container operation failed. " +
- "Container: {} Operation: {} trace ID: {} Error: {}",
- msg.getCreateContainer().getContainerData().getName(),
- msg.getCmdType().name(),
- msg.getTraceID(),
- ex.toString(), ex);
-
- // TODO : Replace with finer error codes.
- return ContainerUtils.getContainerResponse(msg,
- ContainerProtos.Result.CONTAINER_INTERNAL_ERROR,
- ex.toString()).build();
- }
- }
-
- /**
- * Dispatch calls to small file hanlder.
- * @param msg - request
- * @return response
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto smallFileHandler(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- switch (msg.getCmdType()) {
- case PutSmallFile:
- return handlePutSmallFile(msg);
- case GetSmallFile:
- return handleGetSmallFile(msg);
- default:
- return ContainerUtils.unsupportedRequest(msg);
- }
- }
-
- /**
- * Update an existing container with the new container data.
- *
- * @param msg Request
- * @return ContainerCommandResponseProto
- * @throws IOException
- */
- private ContainerCommandResponseProto handleUpdateContainer(
- ContainerCommandRequestProto msg)
- throws IOException {
- if (!msg.hasUpdateContainer()) {
- LOG.debug("Malformed read container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getUpdateContainer().getPipeline());
- String containerName = msg.getUpdateContainer()
- .getContainerData().getName();
-
- ContainerData data = ContainerData.getFromProtBuf(
- msg.getUpdateContainer().getContainerData(), conf);
- boolean forceUpdate = msg.getUpdateContainer().getForceUpdate();
- this.containerManager.updateContainer(
- pipeline, containerName, data, forceUpdate);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return ContainerCommandResponseProto
- * @throws IOException
- */
- private ContainerCommandResponseProto handleReadContainer(
- ContainerCommandRequestProto msg)
- throws IOException {
-
- if (!msg.hasReadContainer()) {
- LOG.debug("Malformed read container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String name = msg.getReadContainer().getName();
- ContainerData container = this.containerManager.readContainer(name);
- return ContainerUtils.getReadContainerResponse(msg, container);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteContainer(
- ContainerCommandRequestProto msg) throws IOException {
-
- if (!msg.hasDeleteContainer()) {
- LOG.debug("Malformed delete container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getDeleteContainer().getPipeline());
- Preconditions.checkNotNull(pipeline);
- String name = msg.getDeleteContainer().getName();
- boolean forceDelete = msg.getDeleteContainer().getForceDelete();
- this.containerManager.deleteContainer(pipeline, name, forceDelete);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * Calls into container logic and returns appropriate response.
- *
- * @param msg - Request
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleCreateContainer(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasCreateContainer()) {
- LOG.debug("Malformed create container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- ContainerData cData = ContainerData.getFromProtBuf(
- msg.getCreateContainer().getContainerData(), conf);
- Preconditions.checkNotNull(cData, "Container data is null");
-
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getCreateContainer().getPipeline());
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-
- this.containerManager.createContainer(pipeline, cData);
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * closes an open container.
- *
- * @param msg -
- * @return
- * @throws IOException
- */
- private ContainerCommandResponseProto handleCloseContainer(
- ContainerCommandRequestProto msg) throws IOException {
- try {
- if (!msg.hasCloseContainer()) {
- LOG.debug("Malformed close Container request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getCloseContainer()
- .getPipeline());
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Attempting to close a closed " +
- "container.", CLOSED_CONTAINER_IO);
- }
- this.containerManager.closeContainer(pipeline.getContainerName());
- return ContainerUtils.getContainerResponse(msg);
- } catch (NoSuchAlgorithmException e) {
- throw new StorageContainerException("No such Algorithm", e,
- NO_SUCH_ALGORITHM);
- }
- }
-
- /**
- * Calls into chunk manager to write a chunk.
- *
- * @param msg - Request.
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleWriteChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasWriteChunk()) {
- LOG.debug("Malformed write chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- String keyName = msg.getWriteChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getWriteChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
-
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getWriteChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
- byte[] data = null;
- if (msg.getWriteChunk().getStage() == ContainerProtos.Stage.WRITE_DATA
- || msg.getWriteChunk().getStage() == ContainerProtos.Stage.COMBINED) {
- data = msg.getWriteChunk().getData().toByteArray();
- metrics.incContainerBytesStats(Type.WriteChunk, data.length);
-
- }
- this.containerManager.getChunkManager()
- .writeChunk(pipeline, keyName, chunkInfo,
- data, msg.getWriteChunk().getStage());
-
- return ChunkUtils.getChunkResponse(msg);
- }
-
- /**
- * Calls into chunk manager to read a chunk.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleReadChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasReadChunk()) {
- LOG.debug("Malformed read chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String keyName = msg.getReadChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getReadChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
-
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getReadChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
- byte[] data = this.containerManager.getChunkManager().readChunk(pipeline,
- keyName, chunkInfo);
- metrics.incContainerBytesStats(Type.ReadChunk, data.length);
- return ChunkUtils.getReadChunkResponse(msg, data, chunkInfo);
- }
-
- /**
- * Calls into chunk manager to write a chunk.
- *
- * @param msg - Request.
- * @return Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteChunk(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasDeleteChunk()) {
- LOG.debug("Malformed delete chunk request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
-
- String keyName = msg.getDeleteChunk().getKeyName();
- Pipeline pipeline = Pipeline.getFromProtoBuf(
- msg.getDeleteChunk().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getDeleteChunk()
- .getChunkData());
- Preconditions.checkNotNull(chunkInfo);
-
- this.containerManager.getChunkManager().deleteChunk(pipeline, keyName,
- chunkInfo);
- return ChunkUtils.getChunkResponse(msg);
- }
-
- /**
- * Put Key handler.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handlePutKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasPutKey()) {
- LOG.debug("Malformed put key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline = Pipeline.getFromProtoBuf(msg.getPutKey().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getPutKey().getKeyData());
- Preconditions.checkNotNull(keyData);
- this.containerManager.getKeyManager().putKey(pipeline, keyData);
- long numBytes = keyData.getProtoBufMessage().toByteArray().length;
- metrics.incContainerBytesStats(Type.PutKey, numBytes);
- return KeyUtils.getKeyResponse(msg);
- }
-
- /**
- * Handle Get Key.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleGetKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasGetKey()) {
- LOG.debug("Malformed get key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getGetKey().getKeyData());
- Preconditions.checkNotNull(keyData);
- KeyData responseData =
- this.containerManager.getKeyManager().getKey(keyData);
- long numBytes = responseData.getProtoBufMessage().toByteArray().length;
- metrics.incContainerBytesStats(Type.GetKey, numBytes);
- return KeyUtils.getKeyDataResponse(msg, responseData);
- }
-
- /**
- * Handle Delete Key.
- *
- * @param msg - Request.
- * @return - Response.
- * @throws IOException
- */
- private ContainerCommandResponseProto handleDeleteKey(
- ContainerCommandRequestProto msg) throws IOException {
- if (!msg.hasDeleteKey()) {
- LOG.debug("Malformed delete key request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getDeleteKey().getPipeline());
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- String keyName = msg.getDeleteKey().getName();
- Preconditions.checkNotNull(keyName);
- Preconditions.checkState(!keyName.isEmpty());
- this.containerManager.getKeyManager().deleteKey(pipeline, keyName);
- return KeyUtils.getKeyResponse(msg);
- }
-
- /**
- * Handles writing a chunk and associated key using single RPC.
- *
- * @param msg - Message.
- * @return ContainerCommandResponseProto
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto handlePutSmallFile(
- ContainerCommandRequestProto msg) throws StorageContainerException {
-
- if (!msg.hasPutSmallFile()) {
- LOG.debug("Malformed put small file request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- try {
-
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getPutSmallFile()
- .getKey().getPipeline());
-
- Preconditions.checkNotNull(pipeline);
- if (!this.containerManager.isOpen(pipeline.getContainerName())) {
- throw new StorageContainerException("Write to closed container.",
- CLOSED_CONTAINER_IO);
- }
- KeyData keyData = KeyData.getFromProtoBuf(msg.getPutSmallFile().getKey()
- .getKeyData());
- ChunkInfo chunkInfo = ChunkInfo.getFromProtoBuf(msg.getPutSmallFile()
- .getChunkInfo());
- byte[] data = msg.getPutSmallFile().getData().toByteArray();
-
- metrics.incContainerBytesStats(Type.PutSmallFile, data.length);
- this.containerManager.getChunkManager().writeChunk(pipeline, keyData
- .getKeyName(), chunkInfo, data, ContainerProtos.Stage.COMBINED);
- List<ContainerProtos.ChunkInfo> chunks = new LinkedList<>();
- chunks.add(chunkInfo.getProtoBufMessage());
- keyData.setChunks(chunks);
- this.containerManager.getKeyManager().putKey(pipeline, keyData);
- return FileUtils.getPutFileResponse(msg);
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException e) {
- throw new StorageContainerException("Put Small File Failed.", e,
- PUT_SMALL_FILE_ERROR);
- }
- }
-
- /**
- * Handles getting a data stream using a key. This helps in reducing the RPC
- * overhead for small files.
- *
- * @param msg - ContainerCommandRequestProto
- * @return ContainerCommandResponseProto
- * @throws StorageContainerException
- */
- private ContainerCommandResponseProto handleGetSmallFile(
- ContainerCommandRequestProto msg) throws StorageContainerException {
- ByteString dataBuf = ByteString.EMPTY;
- if (!msg.hasGetSmallFile()) {
- LOG.debug("Malformed get small file request. trace ID: {}",
- msg.getTraceID());
- return ContainerUtils.malformedRequest(msg);
- }
- try {
- Pipeline pipeline =
- Pipeline.getFromProtoBuf(msg.getGetSmallFile()
- .getKey().getPipeline());
-
- long bytes = 0;
- Preconditions.checkNotNull(pipeline);
- KeyData keyData = KeyData.getFromProtoBuf(msg.getGetSmallFile()
- .getKey().getKeyData());
- KeyData data = this.containerManager.getKeyManager().getKey(keyData);
- ContainerProtos.ChunkInfo c = null;
- for (ContainerProtos.ChunkInfo chunk : data.getChunks()) {
- bytes += chunk.getSerializedSize();
- ByteString current =
- ByteString.copyFrom(this.containerManager.getChunkManager()
- .readChunk(
- pipeline, keyData.getKeyName(), ChunkInfo.getFromProtoBuf(
- chunk)));
- dataBuf = dataBuf.concat(current);
- c = chunk;
- }
- metrics.incContainerBytesStats(Type.GetSmallFile, bytes);
- return FileUtils.getGetSmallFileResponse(msg, dataBuf.toByteArray(),
- ChunkInfo.getFromProtoBuf(c));
- } catch (StorageContainerException e) {
- return ContainerUtils.logAndReturnError(LOG, e, msg);
- } catch (IOException e) {
- throw new StorageContainerException("Get Small File Failed", e,
- GET_SMALL_FILE_ERROR);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
deleted file mode 100644
index 09d9f32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-/*
- This package contains the keyspace manager classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
deleted file mode 100644
index dfa9315..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/Lease.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.apache.hadoop.util.Time;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * This class represents the lease created on a resource. Callback can be
- * registered on the lease which will be executed in case of timeout.
- *
- * @param <T> Resource type for which the lease can be associated
- */
-public class Lease<T> {
-
- /**
- * The resource for which this lease is created.
- */
- private final T resource;
-
- private final long creationTime;
-
- /**
- * Lease lifetime in milliseconds.
- */
- private volatile long leaseTimeout;
-
- private boolean expired;
-
- /**
- * Functions to be called in case of timeout.
- */
- private List<Callable<Void>> callbacks;
-
-
- /**
- * Creates a lease on the specified resource with given timeout.
- *
- * @param resource
- * Resource for which the lease has to be created
- * @param timeout
- * Lease lifetime in milliseconds
- */
- public Lease(T resource, long timeout) {
- this.resource = resource;
- this.leaseTimeout = timeout;
- this.callbacks = Collections.synchronizedList(new ArrayList<>());
- this.creationTime = Time.monotonicNow();
- this.expired = false;
- }
-
- /**
- * Returns true if the lease has expired, else false.
- *
- * @return true if expired, else false
- */
- public boolean hasExpired() {
- return expired;
- }
-
- /**
- * Registers a callback which will be executed in case of timeout. Callbacks
- * are executed in a separate Thread.
- *
- * @param callback
- * The Callable which has to be executed
- * @throws LeaseExpiredException
- * If the lease has already timed out
- */
- public void registerCallBack(Callable<Void> callback)
- throws LeaseExpiredException {
- if(hasExpired()) {
- throw new LeaseExpiredException("Resource: " + resource);
- }
- callbacks.add(callback);
- }
-
- /**
- * Returns the time elapsed since the creation of lease.
- *
- * @return elapsed time in milliseconds
- * @throws LeaseExpiredException
- * If the lease has already timed out
- */
- public long getElapsedTime() throws LeaseExpiredException {
- if(hasExpired()) {
- throw new LeaseExpiredException("Resource: " + resource);
- }
- return Time.monotonicNow() - creationTime;
- }
-
- /**
- * Returns the time available before timeout.
- *
- * @return remaining time in milliseconds
- * @throws LeaseExpiredException
- * If the lease has already timed out
- */
- public long getRemainingTime() throws LeaseExpiredException {
- if(hasExpired()) {
- throw new LeaseExpiredException("Resource: " + resource);
- }
- return leaseTimeout - getElapsedTime();
- }
-
- /**
- * Returns total lease lifetime.
- *
- * @return total lifetime of lease in milliseconds
- * @throws LeaseExpiredException
- * If the lease has already timed out
- */
- public long getLeaseLifeTime() throws LeaseExpiredException {
- if(hasExpired()) {
- throw new LeaseExpiredException("Resource: " + resource);
- }
- return leaseTimeout;
- }
-
- /**
- * Renews the lease timeout period.
- *
- * @param timeout
- * Time to be added to the lease in milliseconds
- * @throws LeaseExpiredException
- * If the lease has already timed out
- */
- public void renew(long timeout) throws LeaseExpiredException {
- if(hasExpired()) {
- throw new LeaseExpiredException("Resource: " + resource);
- }
- leaseTimeout += timeout;
- }
-
- @Override
- public int hashCode() {
- return resource.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if(obj instanceof Lease) {
- return resource.equals(((Lease) obj).resource);
- }
- return false;
- }
-
- @Override
- public String toString() {
- return "Lease<" + resource.toString() + ">";
- }
-
- /**
- * Returns the callbacks to be executed for the lease in case of timeout.
- *
- * @return callbacks to be executed
- */
- List<Callable<Void>> getCallbacks() {
- return callbacks;
- }
-
- /**
- * Expires/Invalidates the lease.
- */
- void invalidate() {
- callbacks = null;
- expired = true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
deleted file mode 100644
index a39ea22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseAlreadyExistException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that there is already a lease acquired on the
- * same resource.
- */
-public class LeaseAlreadyExistException extends LeaseException {
-
- /**
- * Constructs an {@code LeaseAlreadyExistException} with {@code null}
- * as its error detail message.
- */
- public LeaseAlreadyExistException() {
- super();
- }
-
- /**
- * Constructs an {@code LeaseAlreadyExistException} with the specified
- * detail message.
- *
- * @param message
- * The detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method)
- */
- public LeaseAlreadyExistException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
deleted file mode 100644
index 1b7391b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseCallbackExecutor.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.Callable;
-
-/**
- * This class is responsible for executing the callbacks of a lease in case of
- * timeout.
- */
-public class LeaseCallbackExecutor<T> implements Runnable {
-
- private static final Logger LOG = LoggerFactory.getLogger(Lease.class);
-
- private final T resource;
- private final List<Callable<Void>> callbacks;
-
- /**
- * Constructs LeaseCallbackExecutor instance with list of callbacks.
- *
- * @param resource
- * The resource for which the callbacks are executed
- * @param callbacks
- * Callbacks to be executed by this executor
- */
- public LeaseCallbackExecutor(T resource, List<Callable<Void>> callbacks) {
- this.resource = resource;
- this.callbacks = callbacks;
- }
-
- @Override
- public void run() {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Executing callbacks for lease on {}", resource);
- }
- for(Callable<Void> callback : callbacks) {
- try {
- callback.call();
- } catch (Exception e) {
- LOG.warn("Exception while executing callback for lease on {}",
- resource, e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
deleted file mode 100644
index 418f412..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents all lease related exceptions.
- */
-public class LeaseException extends Exception {
-
- /**
- * Constructs an {@code LeaseException} with {@code null}
- * as its error detail message.
- */
- public LeaseException() {
- super();
- }
-
- /**
- * Constructs an {@code LeaseException} with the specified
- * detail message.
- *
- * @param message
- * The detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method)
- */
- public LeaseException(String message) {
- super(message);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
deleted file mode 100644
index 440a023..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseExpiredException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that the lease that is being accessed has expired.
- */
-public class LeaseExpiredException extends LeaseException {
-
- /**
- * Constructs an {@code LeaseExpiredException} with {@code null}
- * as its error detail message.
- */
- public LeaseExpiredException() {
- super();
- }
-
- /**
- * Constructs an {@code LeaseExpiredException} with the specified
- * detail message.
- *
- * @param message
- * The detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method)
- */
- public LeaseExpiredException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
deleted file mode 100644
index b8390dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManager.java
+++ /dev/null
@@ -1,247 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-
-/**
- * LeaseManager is someone who can provide you leases based on your
- * requirement. If you want to return the lease back before it expires,
- * you can give it back to Lease Manager. He is the one responsible for
- * the lifecycle of leases. The resource for which lease is created
- * should have proper {@code equals} method implementation, resource
- * equality is checked while the lease is created.
- *
- * @param <T> Type of leases that this lease manager can create
- */
-public class LeaseManager<T> {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(LeaseManager.class);
-
- private final long defaultTimeout;
- private Map<T, Lease<T>> activeLeases;
- private LeaseMonitor leaseMonitor;
- private Thread leaseMonitorThread;
- private boolean isRunning;
-
- /**
- * Creates an instance of lease manager.
- *
- * @param defaultTimeout
- * Default timeout in milliseconds to be used for lease creation.
- */
- public LeaseManager(long defaultTimeout) {
- this.defaultTimeout = defaultTimeout;
- }
-
- /**
- * Starts the lease manager service.
- */
- public void start() {
- LOG.debug("Starting LeaseManager service");
- activeLeases = new ConcurrentHashMap<>();
- leaseMonitor = new LeaseMonitor();
- leaseMonitorThread = new Thread(leaseMonitor);
- leaseMonitorThread.setName("LeaseManager#LeaseMonitor");
- leaseMonitorThread.setDaemon(true);
- leaseMonitorThread.setUncaughtExceptionHandler((thread, throwable) -> {
- // Let us just restart this thread after logging an error.
- // if this thread is not running we cannot handle Lease expiry.
- LOG.error("LeaseMonitor thread encountered an error. Thread: {}",
- thread.toString(), throwable);
- leaseMonitorThread.start();
- });
- LOG.debug("Starting LeaseManager#LeaseMonitor Thread");
- leaseMonitorThread.start();
- isRunning = true;
- }
-
- /**
- * Returns a lease for the specified resource with default timeout.
- *
- * @param resource
- * Resource for which lease has to be created
- * @throws LeaseAlreadyExistException
- * If there is already a lease on the resource
- */
- public synchronized Lease<T> acquire(T resource)
- throws LeaseAlreadyExistException {
- return acquire(resource, defaultTimeout);
- }
-
- /**
- * Returns a lease for the specified resource with the timeout provided.
- *
- * @param resource
- * Resource for which lease has to be created
- * @param timeout
- * The timeout in milliseconds which has to be set on the lease
- * @throws LeaseAlreadyExistException
- * If there is already a lease on the resource
- */
- public synchronized Lease<T> acquire(T resource, long timeout)
- throws LeaseAlreadyExistException {
- checkStatus();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Acquiring lease on {} for {} milliseconds", resource, timeout);
- }
- if(activeLeases.containsKey(resource)) {
- throw new LeaseAlreadyExistException("Resource: " + resource);
- }
- Lease<T> lease = new Lease<>(resource, timeout);
- activeLeases.put(resource, lease);
- leaseMonitorThread.interrupt();
- return lease;
- }
-
- /**
- * Returns a lease associated with the specified resource.
- *
- * @param resource
- * Resource for which the lease has to be returned
- * @throws LeaseNotFoundException
- * If there is no active lease on the resource
- */
- public Lease<T> get(T resource) throws LeaseNotFoundException {
- checkStatus();
- Lease<T> lease = activeLeases.get(resource);
- if(lease != null) {
- return lease;
- }
- throw new LeaseNotFoundException("Resource: " + resource);
- }
-
- /**
- * Releases the lease associated with the specified resource.
- *
- * @param resource
- * The for which the lease has to be released
- * @throws LeaseNotFoundException
- * If there is no active lease on the resource
- */
- public synchronized void release(T resource)
- throws LeaseNotFoundException {
- checkStatus();
- if(LOG.isDebugEnabled()) {
- LOG.debug("Releasing lease on {}", resource);
- }
- Lease<T> lease = activeLeases.remove(resource);
- if(lease == null) {
- throw new LeaseNotFoundException("Resource: " + resource);
- }
- lease.invalidate();
- }
-
- /**
- * Shuts down the LeaseManager and releases the resources. All the active
- * {@link Lease} will be released (callbacks on leases will not be
- * executed).
- */
- public void shutdown() {
- checkStatus();
- LOG.debug("Shutting down LeaseManager service");
- leaseMonitor.disable();
- leaseMonitorThread.interrupt();
- for(T resource : activeLeases.keySet()) {
- try {
- release(resource);
- } catch(LeaseNotFoundException ex) {
- //Ignore the exception, someone might have released the lease
- }
- }
- isRunning = false;
- }
-
- /**
- * Throws {@link LeaseManagerNotRunningException} if the service is not
- * running.
- */
- private void checkStatus() {
- if(!isRunning) {
- throw new LeaseManagerNotRunningException("LeaseManager not running.");
- }
- }
-
- /**
- * Monitors the leases and expires them based on the timeout, also
- * responsible for executing the callbacks of expired leases.
- */
- private final class LeaseMonitor implements Runnable {
-
- private boolean monitor = true;
- private ExecutorService executorService;
-
- private LeaseMonitor() {
- this.monitor = true;
- this.executorService = Executors.newCachedThreadPool();
- }
-
- @Override
- public void run() {
- while(monitor) {
- LOG.debug("LeaseMonitor: checking for lease expiry");
- long sleepTime = Long.MAX_VALUE;
-
- for (T resource : activeLeases.keySet()) {
- try {
- Lease<T> lease = get(resource);
- long remainingTime = lease.getRemainingTime();
- if (remainingTime <= 0) {
- //Lease has timed out
- List<Callable<Void>> leaseCallbacks = lease.getCallbacks();
- release(resource);
- executorService.execute(
- new LeaseCallbackExecutor(resource, leaseCallbacks));
- } else {
- sleepTime = remainingTime > sleepTime ?
- sleepTime : remainingTime;
- }
- } catch (LeaseNotFoundException | LeaseExpiredException ex) {
- //Ignore the exception, someone might have released the lease
- }
- }
-
- try {
- if(!Thread.interrupted()) {
- Thread.sleep(sleepTime);
- }
- } catch (InterruptedException ignored) {
- // This means a new lease is added to activeLeases.
- }
- }
- }
-
- /**
- * Disables lease monitor, next interrupt call on the thread
- * will stop lease monitor.
- */
- public void disable() {
- monitor = false;
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
deleted file mode 100644
index ced31de..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseManagerNotRunningException.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that there LeaseManager service is not running.
- */
-public class LeaseManagerNotRunningException extends RuntimeException {
-
- /**
- * Constructs an {@code LeaseManagerNotRunningException} with {@code null}
- * as its error detail message.
- */
- public LeaseManagerNotRunningException() {
- super();
- }
-
- /**
- * Constructs an {@code LeaseManagerNotRunningException} with the specified
- * detail message.
- *
- * @param message
- * The detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method)
- */
- public LeaseManagerNotRunningException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
deleted file mode 100644
index c292d33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/LeaseNotFoundException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.lease;
-
-/**
- * This exception represents that the lease that is being accessed does not
- * exist.
- */
-public class LeaseNotFoundException extends LeaseException {
-
- /**
- * Constructs an {@code LeaseNotFoundException} with {@code null}
- * as its error detail message.
- */
- public LeaseNotFoundException() {
- super();
- }
-
- /**
- * Constructs an {@code LeaseNotFoundException} with the specified
- * detail message.
- *
- * @param message
- * The detail message (which is saved for later retrieval
- * by the {@link #getMessage()} method)
- */
- public LeaseNotFoundException(String message) {
- super(message);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
deleted file mode 100644
index 48ee2e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/lease/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-/**
- * A generic lease management API which can be used if a service
- * needs any kind of lease management.
- */
-
-package org.apache.hadoop.ozone.lease;
-/*
- This package contains lease management related classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
deleted file mode 100644
index db399db..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/package-info.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-/**
- This package contains class that support ozone implementation on the datanode
- side.
-
- Main parts of ozone on datanode are:
-
- 1. REST Interface - This code lives under the web directory and listens to the
- WebHDFS port.
-
- 2. Datanode container classes: This support persistence of ozone objects on
- datanode. These classes live under container directory.
-
- 3. Client and Shell: We also support a ozone REST client lib, they are under
- web/client and web/ozShell.
-
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
deleted file mode 100644
index ddcc261..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerDatanodeProtocol.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import java.io.IOException;
-
-/**
- * The protocol spoken between datanodes and SCM. For specifics please the
- * Protoc file that defines this protocol.
- */
-@InterfaceAudience.Private
-public interface StorageContainerDatanodeProtocol {
- /**
- * Returns SCM version.
- * @return Version info.
- */
- SCMVersionResponseProto getVersion(SCMVersionRequestProto versionRequest)
- throws IOException;
-
- /**
- * Used by data node to send a Heartbeat.
- * @param datanodeID - Datanode ID.
- * @param nodeReport - node report state
- * @param reportState - container report state.
- * @return - SCMHeartbeatResponseProto
- * @throws IOException
- */
- SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
- SCMNodeReport nodeReport, ReportState reportState) throws IOException;
-
- /**
- * Register Datanode.
- * @param datanodeID - DatanodID.
- * @param scmAddresses - List of SCMs this datanode is configured to
- * communicate.
- * @return SCM Command.
- */
- SCMRegisteredCmdResponseProto register(DatanodeID datanodeID,
- String[] scmAddresses) throws IOException;
-
- /**
- * Send a container report.
- * @param reports -- Container report.
- * @return container reports response.
- * @throws IOException
- */
- ContainerReportsResponseProto sendContainerReport(
- ContainerReportsRequestProto reports) throws IOException;
-
- /**
- * Used by datanode to send block deletion ACK to SCM.
- * @param request block deletion transactions.
- * @return block deletion transaction response.
- * @throws IOException
- */
- ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto request) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
deleted file mode 100644
index f4a4f32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/StorageContainerNodeProtocol.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-
-import java.util.List;
-
-/**
- * The protocol spoken between datanodes and SCM.
- *
- * Please note that the full protocol spoken between a datanode and SCM is
- * separated into 2 interfaces. One interface that deals with node state and
- * another interface that deals with containers.
- *
- * This interface has functions that deals with the state of datanode.
- */
-@InterfaceAudience.Private
-public interface StorageContainerNodeProtocol {
- /**
- * Gets the version info from SCM.
- * @param versionRequest - version Request.
- * @return - returns SCM version info and other required information needed
- * by datanode.
- */
- VersionResponse getVersion(SCMVersionRequestProto versionRequest);
-
- /**
- * Register the node if the node finds that it is not registered with any SCM.
- * @param datanodeID - Send datanodeID with Node info, but datanode UUID is
- * empty. Server returns a datanodeID for the given node.
- * @return SCMHeartbeatResponseProto
- */
- SCMCommand register(DatanodeID datanodeID);
-
- /**
- * Send heartbeat to indicate the datanode is alive and doing well.
- * @param datanodeID - Datanode ID.
- * @param nodeReport - node report.
- * @param reportState - container report.
- * @return SCMheartbeat response list
- */
- List<SCMCommand> sendHeartbeat(DatanodeID datanodeID,
- SCMNodeReport nodeReport, ReportState reportState);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
deleted file mode 100644
index 8d8f4f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/VersionResponse.java
+++ /dev/null
@@ -1,150 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * Version response class.
- */
-public class VersionResponse {
- private final int version;
- private final Map<String, String> values;
-
- /**
- * Creates a version response class.
- * @param version
- * @param values
- */
- public VersionResponse(int version, Map<String, String> values) {
- this.version = version;
- this.values = values;
- }
-
- /**
- * Creates a version Response class.
- * @param version
- */
- public VersionResponse(int version) {
- this.version = version;
- this.values = new HashMap<>();
- }
-
- /**
- * Returns a new Builder.
- * @return - Builder.
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Returns this class from protobuf message.
- * @param response - SCMVersionResponseProto
- * @return VersionResponse
- */
- public static VersionResponse getFromProtobuf(SCMVersionResponseProto
- response) {
- return new VersionResponse(response.getSoftwareVersion(),
- response.getKeysList().stream()
- .collect(Collectors.toMap(KeyValue::getKey,
- KeyValue::getValue)));
- }
-
- /**
- * Adds a value to version Response.
- * @param key - String
- * @param value - String
- */
- public void put(String key, String value) {
- if (this.values.containsKey(key)) {
- throw new IllegalArgumentException("Duplicate key in version response");
- }
- values.put(key, value);
- }
-
- /**
- * Return a protobuf message.
- * @return SCMVersionResponseProto.
- */
- public SCMVersionResponseProto getProtobufMessage() {
-
- List<KeyValue> list = new LinkedList<>();
- for (Map.Entry<String, String> entry : values.entrySet()) {
- list.add(KeyValue.newBuilder().setKey(entry.getKey()).
- setValue(entry.getValue()).build());
- }
- return
- SCMVersionResponseProto.newBuilder()
- .setSoftwareVersion(this.version)
- .addAllKeys(list).build();
- }
-
- /**
- * Builder class.
- */
- public static class Builder {
- private int version;
- private Map<String, String> values;
-
- Builder() {
- values = new HashMap<>();
- }
-
- /**
- * Sets the version.
- * @param ver - version
- * @return Builder
- */
- public Builder setVersion(int ver) {
- this.version = ver;
- return this;
- }
-
- /**
- * Adds a value to version Response.
- * @param key - String
- * @param value - String
- */
- public Builder addValue(String key, String value) {
- if (this.values.containsKey(key)) {
- throw new IllegalArgumentException("Duplicate key in version response");
- }
- values.put(key, value);
- return this;
- }
-
- /**
- * Builds the version response.
- * @return VersionResponse.
- */
- public VersionResponse build() {
- return new VersionResponse(this.version, this.values);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
deleted file mode 100644
index 79aa132..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/CloseContainerCommand.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMCloseContainerCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-import static org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type.closeContainerCommand;
-
-/**
- * Asks datanode to close a container.
- */
-public class CloseContainerCommand
- extends SCMCommand<SCMCloseContainerCmdResponseProto> {
-
- private String containerName;
-
- public CloseContainerCommand(String containerName) {
- this.containerName = containerName;
- }
-
- /**
- * Returns the type of this command.
- *
- * @return Type
- */
- @Override
- public Type getType() {
- return closeContainerCommand;
- }
-
- /**
- * Gets the protobuf message of this object.
- *
- * @return A protobuf message.
- */
- @Override
- public byte[] getProtoBufMessage() {
- return getProto().toByteArray();
- }
-
- public SCMCloseContainerCmdResponseProto getProto() {
- return SCMCloseContainerCmdResponseProto.newBuilder()
- .setContainerName(containerName).build();
- }
-
- public static CloseContainerCommand getFromProtobuf(
- SCMCloseContainerCmdResponseProto closeContainerProto) {
- Preconditions.checkNotNull(closeContainerProto);
- return new CloseContainerCommand(closeContainerProto.getContainerName());
-
- }
-
- public String getContainerName() {
- return containerName;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
deleted file mode 100644
index 8e3463d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/DeleteBlocksCommand.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMDeleteBlocksCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-
-import java.util.List;
-
-/**
- * A SCM command asks a datanode to delete a number of blocks.
- */
-public class DeleteBlocksCommand extends
- SCMCommand<SCMDeleteBlocksCmdResponseProto> {
-
- private List<DeletedBlocksTransaction> blocksTobeDeleted;
-
-
- public DeleteBlocksCommand(List<DeletedBlocksTransaction> blocks) {
- this.blocksTobeDeleted = blocks;
- }
-
- public List<DeletedBlocksTransaction> blocksTobeDeleted() {
- return this.blocksTobeDeleted;
- }
-
- @Override
- public Type getType() {
- return Type.deleteBlocksCommand;
- }
-
- @Override
- public byte[] getProtoBufMessage() {
- return getProto().toByteArray();
- }
-
- public static DeleteBlocksCommand getFromProtobuf(
- SCMDeleteBlocksCmdResponseProto deleteBlocksProto) {
- return new DeleteBlocksCommand(deleteBlocksProto
- .getDeletedBlocksTransactionsList());
- }
-
- public SCMDeleteBlocksCmdResponseProto getProto() {
- return SCMDeleteBlocksCmdResponseProto.newBuilder()
- .addAllDeletedBlocksTransactions(blocksTobeDeleted).build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
deleted file mode 100644
index bf430ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/RegisteredCommand.java
+++ /dev/null
@@ -1,177 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
- .ErrorCode;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-
-/**
- * Response to Datanode Register call.
- */
-public class RegisteredCommand extends
- SCMCommand<SCMRegisteredCmdResponseProto> {
- private String datanodeUUID;
- private String clusterID;
- private ErrorCode error;
-
- public RegisteredCommand(final ErrorCode error, final String datanodeUUID,
- final String clusterID) {
- this.datanodeUUID = datanodeUUID;
- this.clusterID = clusterID;
- this.error = error;
- }
-
- /**
- * Returns a new builder.
- *
- * @return - Builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Returns the type of this command.
- *
- * @return Type
- */
- @Override
- public Type getType() {
- return Type.registeredCommand;
- }
-
- /**
- * Returns datanode UUID.
- *
- * @return - Datanode ID.
- */
- public String getDatanodeUUID() {
- return datanodeUUID;
- }
-
- /**
- * Returns cluster ID.
- *
- * @return -- ClusterID
- */
- public String getClusterID() {
- return clusterID;
- }
-
- /**
- * Returns ErrorCode.
- *
- * @return - ErrorCode
- */
- public ErrorCode getError() {
- return error;
- }
-
- /**
- * Gets the protobuf message of this object.
- *
- * @return A protobuf message.
- */
- @Override
- public byte[] getProtoBufMessage() {
- return SCMRegisteredCmdResponseProto.newBuilder()
- .setClusterID(this.clusterID)
- .setDatanodeUUID(this.datanodeUUID)
- .setErrorCode(this.error)
- .build().toByteArray();
- }
-
- /**
- * A builder class to verify all values are sane.
- */
- public static class Builder {
- private String datanodeUUID;
- private String clusterID;
- private ErrorCode error;
-
- /**
- * sets UUID.
- *
- * @param dnUUID - datanode UUID
- * @return Builder
- */
- public Builder setDatanodeUUID(String dnUUID) {
- this.datanodeUUID = dnUUID;
- return this;
- }
-
- /**
- * Create this object from a Protobuf message.
- *
- * @param response - RegisteredCmdResponseProto
- * @return RegisteredCommand
- */
- public RegisteredCommand getFromProtobuf(SCMRegisteredCmdResponseProto
- response) {
- Preconditions.checkNotNull(response);
- return new RegisteredCommand(response.getErrorCode(),
- response.hasDatanodeUUID() ? response.getDatanodeUUID(): "",
- response.hasClusterID() ? response.getClusterID(): "");
- }
-
- /**
- * Sets cluster ID.
- *
- * @param cluster - clusterID
- * @return Builder
- */
- public Builder setClusterID(String cluster) {
- this.clusterID = cluster;
- return this;
- }
-
- /**
- * Sets Error code.
- *
- * @param errorCode - error code
- * @return Builder
- */
- public Builder setErrorCode(ErrorCode errorCode) {
- this.error = errorCode;
- return this;
- }
-
- /**
- * Build the command object.
- *
- * @return RegisteredCommand
- */
- public RegisteredCommand build() {
- if ((this.error == ErrorCode.success) &&
- (this.datanodeUUID == null || this.datanodeUUID.isEmpty()) ||
- (this.clusterID == null || this.clusterID.isEmpty())) {
- throw new IllegalArgumentException("On success, RegisteredCommand " +
- "needs datanodeUUID and ClusterID.");
- }
-
- return new
- RegisteredCommand(this.error, this.datanodeUUID, this.clusterID);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
deleted file mode 100644
index f87d35b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/ReregisterCommand.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-
-import static org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMReregisterCmdResponseProto;
-import static org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type.reregisterCommand;
-
-/**
- * Informs a datanode to register itself with SCM again.
- */
-public class ReregisterCommand extends
- SCMCommand<SCMReregisterCmdResponseProto>{
-
- /**
- * Returns the type of this command.
- *
- * @return Type
- */
- @Override
- public Type getType() {
- return reregisterCommand;
- }
-
- /**
- * Gets the protobuf message of this object.
- *
- * @return A protobuf message.
- */
- @Override
- public byte[] getProtoBufMessage() {
- return getProto().toByteArray();
- }
-
- public SCMReregisterCmdResponseProto getProto() {
- return SCMReregisterCmdResponseProto
- .newBuilder()
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
deleted file mode 100644
index fe9b12d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SCMCommand.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-import com.google.protobuf.GeneratedMessage;
-
-/**
- * A class that acts as the base class to convert between Java and SCM
- * commands in protobuf format.
- * @param <T>
- */
-public abstract class SCMCommand<T extends GeneratedMessage> {
- /**
- * Returns the type of this command.
- * @return Type
- */
- public abstract Type getType();
-
- /**
- * Gets the protobuf message of this object.
- * @return A protobuf message.
- */
- public abstract byte[] getProtoBufMessage();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
deleted file mode 100644
index 6a9fc44..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/SendContainerCommand.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol.commands;
-
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SendContainerReportProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.Type;
-
-/**
- * Allows a Datanode to send in the container report.
- */
-public class SendContainerCommand extends SCMCommand<SendContainerReportProto> {
- /**
- * Returns a NullCommand class from NullCommandResponse Proto.
- * @param unused - unused
- * @return NullCommand
- */
- public static SendContainerCommand getFromProtobuf(
- final SendContainerReportProto unused) {
- return new SendContainerCommand();
- }
-
- /**
- * returns a new builder.
- * @return Builder
- */
- public static SendContainerCommand.Builder newBuilder() {
- return new SendContainerCommand.Builder();
- }
-
- /**
- * Returns the type of this command.
- *
- * @return Type
- */
- @Override
- public Type getType() {
- return Type.sendContainerReport;
- }
-
- /**
- * Gets the protobuf message of this object.
- *
- * @return A protobuf message.
- */
- @Override
- public byte[] getProtoBufMessage() {
- return SendContainerReportProto.newBuilder().build().toByteArray();
- }
-
- /**
- * A Builder class this is the standard pattern we are using for all commands.
- */
- public static class Builder {
- /**
- * Return a null command.
- * @return - NullCommand.
- */
- public SendContainerCommand build() {
- return new SendContainerCommand();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
deleted file mode 100644
index 7083c1b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/commands/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocol.commands;
-/**
- Set of classes that help in protoc conversions.
- **/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
deleted file mode 100644
index 8bc29aa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocol/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocol;
-
-/**
- * This package contains classes for Ozone protocol definitions.
- */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
deleted file mode 100644
index 1fe9a18..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/OzoneMetadataManager.java
+++ /dev/null
@@ -1,1138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.localstorage;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.response.VolumeOwner;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.charset.Charset;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.ListIterator;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * A stand alone Ozone implementation that allows us to run Ozone tests in local
- * mode. This acts as the ozone backend when using MiniDFSCluster for testing.
- */
-public final class OzoneMetadataManager {
-
- /*
- OzoneMetadataManager manages volume/bucket/object metadata and
- data.
-
- Metadata is maintained in 2 level DB files, UserDB and MetadataDB.
-
- UserDB contains a Name and a List. For example volumes owned by the user
- bilbo, would be maintained in UserDB as {bilbo}->{shire, rings}
-
- This list part of mapping is context sensitive. That is, if you use {user
- name} as the key, the list you get is a list of volumes. if you use
- {user/volume} as the key the list you get is list of buckets. if you use
- {user/volume/bucket} as key the list you get is the list of objects.
-
- All keys in the UserDB starts with the UserName.
-
- We also need to maintain a flat namespace for volumes. This is
- maintained by the MetadataDB. MetadataDB contains the name of an
- object(volume, bucket or key) and its associated metadata.
- The keys in the Metadata DB are {volume}, {volume/bucket} or
- {volume/bucket/key}. User name is absent, so we have a common root name
- space for the volume.
-
- The value of part of metadataDB points to corresponding *Info structures.
- {volume] -> volumeInfo
- {volume/bucket} -> bucketInfo
- {volume/bucket/key} -> keyInfo
-
-
- Here are various work flows :
-
- CreateVolume -> Check if Volume exists in metadataDB, if not update UserDB
- with a list of volumes and update metadataDB with VolumeInfo.
-
- DeleteVolume -> Check the Volume, and check the VolumeInfo->bucketCount.
- if bucketCount == 0, delete volume from userDB->{List of volumes} and
- metadataDB.
-
- Very similar work flows exist for CreateBucket and DeleteBucket.
-
- // Please note : These database operations are *not* transactional,
- // which means that failure can lead to inconsistencies.
- // Only way to recover is to reset to a clean state, or
- // use rm -rf /tmp/ozone :)
-
- We have very simple locking policy. We have a ReaderWriter lock that is
- taken for each action, this lock is aptly named "lock".
-
- All actions *must* be performed with a lock held, either a read
- lock or a write lock. Violation of these locking policies can be harmful.
-
-
- // // IMPORTANT :
- // // This is a simulation layer, this is NOT how the real
- // // OZONE functions. This is written to so that we can write
- // // stand-alone tests for the protocol and client code.
-
-*/
- static final Logger LOG = LoggerFactory.getLogger(OzoneMetadataManager.class);
- private static final String USER_DB = "/user.db";
- private static final String META_DB = "/metadata.db";
- private static OzoneMetadataManager bm = null;
- private MetadataStore userDB;
- private MetadataStore metadataDB;
- private ReadWriteLock lock;
- private Charset encoding = Charset.forName("UTF-8");
- private String storageRoot;
- private static final String OBJECT_DIR = "/_objects/";
-
- // This table keeps a pointer to objects whose operations
- // are in progress but not yet committed to persistent store
- private ConcurrentHashMap<OutputStream, String> inProgressObjects;
-
- /**
- * Constructs OzoneMetadataManager.
- */
- private OzoneMetadataManager(Configuration conf) throws IOException {
-
- lock = new ReentrantReadWriteLock();
- storageRoot =
- conf.getTrimmed(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
- OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
-
- File file = new File(storageRoot + OBJECT_DIR);
-
- if (!file.exists() && !file.mkdirs()) {
- LOG.error("Creation of Ozone root failed. " + file.toString());
- throw new IOException("Creation of Ozone root failed.");
- }
-
- try {
- userDB = MetadataStoreBuilder.newBuilder()
- .setDbFile(new File(storageRoot + USER_DB))
- .setCreateIfMissing(true)
- .build();
- metadataDB = MetadataStoreBuilder.newBuilder()
- .setDbFile(new File(storageRoot + META_DB))
- .setCreateIfMissing(true)
- .build();
- inProgressObjects = new ConcurrentHashMap<>();
- } catch (IOException ex) {
- LOG.error("Cannot open db :" + ex.getMessage());
- throw ex;
- }
- }
-
- /**
- * Gets Ozone Manager.
- *
- * @return OzoneMetadataManager
- */
- public static synchronized OzoneMetadataManager
- getOzoneMetadataManager(Configuration conf) throws IOException {
- if (bm == null) {
- bm = new OzoneMetadataManager(conf);
- }
- return bm;
- }
-
- /**
- * Creates a volume.
- *
- * @param args - VolumeArgs
- * @throws OzoneException
- */
- public void createVolume(VolumeArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- SimpleDateFormat format =
- new SimpleDateFormat(OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
- format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-
- byte[] volumeName =
- metadataDB.get(args.getVolumeName().getBytes(encoding));
-
- if (volumeName != null) {
- LOG.debug("Volume {} already exists.", volumeName);
- throw ErrorTable.newError(ErrorTable.VOLUME_ALREADY_EXISTS, args);
- }
-
- VolumeInfo newVInfo = new VolumeInfo(args.getVolumeName(), format
- .format(new Date(System.currentTimeMillis())), args.getAdminName());
-
- newVInfo.setQuota(args.getQuota());
- VolumeOwner owner = new VolumeOwner(args.getUserName());
- newVInfo.setOwner(owner);
-
- ListVolumes volumeList;
- byte[] userVolumes = userDB.get(args.getUserName().getBytes(encoding));
- if (userVolumes == null) {
- volumeList = new ListVolumes();
- } else {
- volumeList = ListVolumes.parse(new String(userVolumes, encoding));
- }
-
- volumeList.addVolume(newVInfo);
- volumeList.sort();
-
- // Please note : These database operations are *not* transactional,
- // which means that failure can lead to inconsistencies.
- // Only way to recover is to reset to a clean state, or
- // use rm -rf /tmp/ozone :)
-
-
- userDB.put(args.getUserName().getBytes(encoding),
- volumeList.toDBString().getBytes(encoding));
-
- metadataDB.put(args.getVolumeName().getBytes(encoding),
- newVInfo.toDBString().getBytes(encoding));
-
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Updates the Volume properties like Owner Name and Quota.
- *
- * @param args - Volume Args
- * @param property - Flag which tells us what property to upgrade
- * @throws OzoneException
- */
- public void setVolumeProperty(VolumeArgs args, VolumeProperty property)
- throws OzoneException {
- lock.writeLock().lock();
- try {
- byte[] volumeInfo =
- metadataDB.get(args.getVolumeName().getBytes(encoding));
- if (volumeInfo == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
- }
- VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding));
-
- byte[] userBytes = userDB.get(args.getResourceName().getBytes(encoding));
- ListVolumes volumeList;
- if (userBytes == null) {
- volumeList = new ListVolumes();
- } else {
- volumeList = ListVolumes.parse(new String(userBytes, encoding));
- }
-
- switch (property) {
- case OWNER:
- // needs new owner, we delete the volume object from the
- // old user's volume list
- removeOldOwner(info);
- VolumeOwner owner = new VolumeOwner(args.getUserName());
- // set the new owner
- info.setOwner(owner);
- break;
- case QUOTA:
- // if this is quota update we just remove the old object from the
- // current users list and update the same object later.
- volumeList.getVolumes().remove(info);
- info.setQuota(args.getQuota());
- break;
- default:
- OzoneException ozEx =
- ErrorTable.newError(ErrorTable.BAD_PROPERTY, args);
- ozEx.setMessage("Volume property is not recognized");
- throw ozEx;
- }
-
- volumeList.addVolume(info);
-
- metadataDB.put(args.getVolumeName().getBytes(encoding),
- info.toDBString().getBytes(encoding));
-
- // if this is an owner change this put will create a new owner or update
- // the owner's volume list.
- userDB.put(args.getResourceName().getBytes(encoding),
- volumeList.toDBString().getBytes(encoding));
-
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Removes the old owner from the volume.
- *
- * @param info - VolumeInfo
- * @throws IOException
- */
- private void removeOldOwner(VolumeInfo info) throws IOException {
- // We need to look the owner that we know is the current owner
- byte[] volumeBytes =
- userDB.get(info.getOwner().getName().getBytes(encoding));
- ListVolumes volumeList =
- ListVolumes.parse(new String(volumeBytes, encoding));
- volumeList.getVolumes().remove(info);
-
- // Write the new list info to the old user data
- userDB.put(info.getOwner().getName().getBytes(encoding),
- volumeList.toDBString().getBytes(encoding));
- }
-
- /**
- * Checks if you are the owner of a specific volume.
- *
- * @param volume - Volume Name whose access permissions needs to be checked
- * @param acl - requested acls which needs to be checked for access
- * @return - True if you are the owner, false otherwise
- * @throws OzoneException
- */
- public boolean checkVolumeAccess(String volume, OzoneAcl acl)
- throws OzoneException {
- lock.readLock().lock();
- try {
- byte[] volumeInfo =
- metadataDB.get(volume.getBytes(encoding));
- if (volumeInfo == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, null);
- }
-
- VolumeInfo info = VolumeInfo.parse(new String(volumeInfo, encoding));
- return info.getOwner().getName().equals(acl.getName());
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, null, ex);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * getVolumeInfo returns the Volume Info of a specific volume.
- *
- * @param args - Volume args
- * @return VolumeInfo
- * @throws OzoneException
- */
- public VolumeInfo getVolumeInfo(VolumeArgs args) throws OzoneException {
- lock.readLock().lock();
- try {
- byte[] volumeInfo =
- metadataDB.get(args.getVolumeName().getBytes(encoding));
- if (volumeInfo == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
- }
-
- return VolumeInfo.parse(new String(volumeInfo, encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Returns all the volumes owned by a specific user.
- *
- * @param args - User Args
- * @return - ListVolumes
- * @throws OzoneException
- */
- public ListVolumes listVolumes(ListArgs args) throws OzoneException {
- lock.readLock().lock();
- try {
- if (args.isRootScan()) {
- return listAllVolumes(args);
- }
-
- UserArgs uArgs = (UserArgs) args.getArgs();
- byte[] volumeList = userDB.get(uArgs.getUserName().getBytes(encoding));
- if (volumeList == null) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs);
- }
-
- String prefix = args.getPrefix();
- int maxCount = args.getMaxKeys();
- String prevKey = args.getPrevKey();
- if (prevKey != null) {
- // Format is username/volumeName, in local mode we don't use the
- // user name since we have a userName DB.
- String[] volName = args.getPrevKey().split("/");
- if (volName.length < 2) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, uArgs);
- }
- prevKey = volName[1];
- }
- return getFilteredVolumes(volumeList, prefix, prevKey, maxCount);
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Returns a List of Volumes that meet the prefix, prevkey and maxCount
- * constraints.
- *
- * @param volumeList - Byte Array of Volume Info.
- * @param prefix - prefix string.
- * @param prevKey - PrevKey
- * @param maxCount - Maximum Count.
- * @return ListVolumes.
- * @throws IOException
- */
- private ListVolumes getFilteredVolumes(byte[] volumeList, String prefix,
- String prevKey, int maxCount) throws
- IOException {
- ListVolumes volumes = ListVolumes.parse(new String(volumeList,
- encoding));
- int currentCount = 0;
- ListIterator<VolumeInfo> iter = volumes.getVolumes().listIterator();
- ListVolumes filteredVolumes = new ListVolumes();
- while (currentCount < maxCount && iter.hasNext()) {
- VolumeInfo vInfo = iter.next();
- if (isMatchingPrefix(prefix, vInfo) && isAfterKey(prevKey, vInfo)) {
- filteredVolumes.addVolume(vInfo);
- currentCount++;
- }
- }
- return filteredVolumes;
- }
-
- /**
- * Returns all volumes in a cluster.
- *
- * @param args - ListArgs.
- * @return ListVolumes.
- * @throws OzoneException
- */
- public ListVolumes listAllVolumes(ListArgs args)
- throws OzoneException, IOException {
- String prefix = args.getPrefix();
- final String prevKey;
- int maxCount = args.getMaxKeys();
- String userName = null;
-
- if (args.getPrevKey() != null) {
- // Format is username/volumeName
- String[] volName = args.getPrevKey().split("/");
- if (volName.length < 2) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs());
- }
-
- byte[] userNameBytes = userDB.get(volName[0].getBytes(encoding));
- userName = new String(userNameBytes, encoding);
- prevKey = volName[1];
- } else {
- userName = new String(userDB.peekAround(0, null).getKey(), encoding);
- prevKey = null;
- }
-
- if (userName == null || userName.isEmpty()) {
- throw ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs());
- }
-
- ListVolumes returnSet = new ListVolumes();
- // we need to iterate through users until we get maxcount volumes
- // or no more volumes are left.
- userDB.iterate(null, (key, value) -> {
- int currentSize = returnSet.getVolumes().size();
- if (currentSize < maxCount) {
- String name = new String(key, encoding);
- byte[] volumeList = userDB.get(name.getBytes(encoding));
- if (volumeList == null) {
- throw new IOException(
- ErrorTable.newError(ErrorTable.USER_NOT_FOUND, args.getArgs()));
- }
- returnSet.getVolumes().addAll(
- getFilteredVolumes(volumeList, prefix, prevKey,
- maxCount - currentSize).getVolumes());
- return true;
- } else {
- return false;
- }
- });
-
- return returnSet;
- }
-
- /**
- * Checks if a name starts with a matching prefix.
- *
- * @param prefix - prefix string.
- * @param vInfo - volume info.
- * @return true or false.
- */
- private boolean isMatchingPrefix(String prefix, VolumeInfo vInfo) {
- if (prefix == null || prefix.isEmpty()) {
- return true;
- }
- return vInfo.getVolumeName().startsWith(prefix);
- }
-
- /**
- * Checks if the key is after the prevKey.
- *
- * @param prevKey - String prevKey.
- * @param vInfo - volume Info.
- * @return - true or false.
- */
- private boolean isAfterKey(String prevKey, VolumeInfo vInfo) {
- if (prevKey == null || prevKey.isEmpty()) {
- return true;
- }
- return prevKey.compareTo(vInfo.getVolumeName()) < 0;
- }
-
- /**
- * Deletes a volume if it exists and is empty.
- *
- * @param args - volume args
- * @throws OzoneException
- */
- public void deleteVolume(VolumeArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- byte[] volumeName =
- metadataDB.get(args.getVolumeName().getBytes(encoding));
- if (volumeName == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
- }
-
- VolumeInfo vInfo = VolumeInfo.parse(new String(volumeName, encoding));
-
- // Only remove volumes if they are empty.
- if (vInfo.getBucketCount() > 0) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_EMPTY, args);
- }
-
- ListVolumes volumeList;
- String user = vInfo.getOwner().getName();
- byte[] userVolumes = userDB.get(user.getBytes(encoding));
- if (userVolumes == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
- }
-
- volumeList = ListVolumes.parse(new String(userVolumes, encoding));
- volumeList.getVolumes().remove(vInfo);
-
- metadataDB.delete(args.getVolumeName().getBytes(encoding));
- userDB.put(user.getBytes(encoding),
- volumeList.toDBString().getBytes(encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Create a bucket if it does not exist.
- *
- * @param args - BucketArgs
- * @throws OzoneException
- */
- public void createBucket(BucketArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- // check if volume exists, buckets cannot be created without volumes
- byte[] volumeName = metadataDB.get(args.getVolumeName()
- .getBytes(encoding));
- if (volumeName == null) {
- throw ErrorTable.newError(ErrorTable.VOLUME_NOT_FOUND, args);
- }
-
- // A resource name is volume/bucket -- That is the key in metadata table
- byte[] bucketName = metadataDB.get(args.getResourceName()
- .getBytes(encoding));
- if (bucketName != null) {
- throw ErrorTable.newError(ErrorTable.BUCKET_ALREADY_EXISTS, args);
- }
-
- BucketInfo bucketInfo =
- new BucketInfo(args.getVolumeName(), args.getBucketName());
-
- if (args.getRemoveAcls() != null) {
- OzoneException ex = ErrorTable.newError(ErrorTable.MALFORMED_ACL, args);
- ex.setMessage("Remove ACLs specified in bucket create. Please remove "
- + "them and retry.");
- throw ex;
- }
-
- VolumeInfo volInfo = VolumeInfo.parse(new String(volumeName, encoding));
- volInfo.setBucketCount(volInfo.getBucketCount() + 1);
-
- bucketInfo.setAcls(args.getAddAcls());
- bucketInfo.setStorageType(args.getStorageType());
- bucketInfo.setVersioning(args.getVersioning());
- ListBuckets bucketList;
-
- // get bucket list from user/volume -> bucketList
- byte[] volumeBuckets = userDB.get(args.getParentName()
- .getBytes(encoding));
- if (volumeBuckets == null) {
- bucketList = new ListBuckets();
- } else {
- bucketList = ListBuckets.parse(new String(volumeBuckets, encoding));
- }
-
- bucketList.addBucket(bucketInfo);
- bucketList.sort();
-
- // Update Volume->bucketCount
- userDB.put(args.getVolumeName().getBytes(encoding),
- volInfo.toDBString().getBytes(encoding));
-
- // Now update the userDB with user/volume -> bucketList
- userDB.put(args.getParentName().getBytes(encoding),
- bucketList.toDBString().getBytes(encoding));
-
- // Update userDB with volume/bucket -> empty key list
- userDB.put(args.getResourceName().getBytes(encoding),
- new ListKeys().toDBString().getBytes(encoding));
-
- // and update the metadataDB with volume/bucket->BucketInfo
- metadataDB.put(args.getResourceName().getBytes(encoding),
- bucketInfo.toDBString().getBytes(encoding));
-
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Updates the Bucket properties like ACls and Storagetype.
- *
- * @param args - Bucket Args
- * @param property - Flag which tells us what property to upgrade
- * @throws OzoneException
- */
- public void setBucketProperty(BucketArgs args, BucketProperty property)
- throws OzoneException {
-
- lock.writeLock().lock();
- try {
- // volume/bucket-> bucketInfo
- byte[] bucketInfo = metadataDB.get(args.getResourceName().
- getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- BucketInfo info = BucketInfo.parse(new String(bucketInfo, encoding));
- byte[] volumeBuckets = userDB.get(args.getParentName()
- .getBytes(encoding));
- ListBuckets bucketList = ListBuckets.parse(new String(volumeBuckets,
- encoding));
- bucketList.getBuckets().remove(info);
-
- switch (property) {
- case ACLS:
- processRemoveAcls(args, info);
- processAddAcls(args, info);
- break;
- case STORAGETYPE:
- info.setStorageType(args.getStorageType());
- break;
- case VERSIONING:
- info.setVersioning(args.getVersioning());
- break;
- default:
- OzoneException ozEx =
- ErrorTable.newError(ErrorTable.BAD_PROPERTY, args);
- ozEx.setMessage("Bucket property is not recognized.");
- throw ozEx;
- }
-
- bucketList.addBucket(info);
- metadataDB.put(args.getResourceName().getBytes(encoding),
- info.toDBString().getBytes(encoding));
-
- userDB.put(args.getParentName().getBytes(encoding),
- bucketList.toDBString().getBytes(encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Process Remove Acls and remove them from the bucket.
- *
- * @param args - BucketArgs
- * @param info - BucketInfo
- */
- private void processRemoveAcls(BucketArgs args, BucketInfo info) {
- List<OzoneAcl> removeAcls = args.getRemoveAcls();
- if ((removeAcls == null) || (info.getAcls() == null)) {
- return;
- }
- for (OzoneAcl racl : args.getRemoveAcls()) {
- ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator();
- while (aclIter.hasNext()) {
- if (racl.equals(aclIter.next())) {
- aclIter.remove();
- break;
- }
- }
- }
- }
-
- /**
- * Process Add Acls and Add them to the bucket.
- *
- * @param args - BucketArgs
- * @param info - BucketInfo
- */
- private void processAddAcls(BucketArgs args, BucketInfo info) {
- List<OzoneAcl> addAcls = args.getAddAcls();
- if ((addAcls == null)) {
- return;
- }
-
- if (info.getAcls() == null) {
- info.setAcls(addAcls);
- return;
- }
-
- for (OzoneAcl newacl : addAcls) {
- ListIterator<OzoneAcl> aclIter = info.getAcls().listIterator();
- while (aclIter.hasNext()) {
- if (newacl.equals(aclIter.next())) {
- continue;
- }
- }
- info.getAcls().add(newacl);
- }
- }
-
- /**
- * Deletes a given bucket.
- *
- * @param args - BucketArgs
- * @throws OzoneException
- */
- public void deleteBucket(BucketArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- byte[] bucketInfo = metadataDB.get(args.getResourceName()
- .getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
-
- // Only remove buckets if they are empty.
- if (bInfo.getKeyCount() > 0) {
- throw ErrorTable.newError(ErrorTable.BUCKET_NOT_EMPTY, args);
- }
-
- byte[] bucketBytes = userDB.get(args.getParentName().getBytes(encoding));
- if (bucketBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- ListBuckets bucketList =
- ListBuckets.parse(new String(bucketBytes, encoding));
- bucketList.getBuckets().remove(bInfo);
-
- metadataDB.delete(args.getResourceName().getBytes(encoding));
- userDB.put(args.getParentName().getBytes(encoding),
- bucketList.toDBString().getBytes(encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Returns the Bucket info for a given bucket.
- *
- * @param args - Bucket Args
- * @return BucketInfo - Bucket Information
- * @throws OzoneException
- */
- public BucketInfo getBucketInfo(BucketArgs args) throws OzoneException {
- lock.readLock().lock();
- try {
- byte[] bucketBytes = metadataDB.get(args.getResourceName()
- .getBytes(encoding));
- if (bucketBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- return BucketInfo.parse(new String(bucketBytes, encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Returns a list of buckets for a given volume.
- *
- * @param args - volume args
- * @return List of buckets
- * @throws OzoneException
- */
- public ListBuckets listBuckets(ListArgs args) throws OzoneException {
- lock.readLock().lock();
- try {
- Preconditions.checkState(args.getArgs() instanceof VolumeArgs);
- VolumeArgs vArgs = (VolumeArgs) args.getArgs();
- String userVolKey = vArgs.getUserName() + "/" + vArgs.getVolumeName();
-
- // TODO : Query using Prefix and PrevKey
- byte[] bucketBytes = userDB.get(userVolKey.getBytes(encoding));
- if (bucketBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_VOLUME_NAME,
- args.getArgs());
- }
- return ListBuckets.parse(new String(bucketBytes, encoding));
- } catch (IOException ex) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args.getArgs(), ex);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Creates a key and returns a stream to which this key can be written to.
- *
- * @param args KeyArgs
- * @return - A stream into which key can be written to.
- * @throws OzoneException
- */
- public OutputStream createKey(KeyArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
-
- // Please don't try trillion objects unless the physical file system
- // is capable of doing that in a single directory.
-
- String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
- File f = new File(fullPath);
-
- // In real ozone it would not be this way, a file will be overwritten
- // only if the upload is successful.
- if (f.exists()) {
- LOG.debug("we are overwriting a file. This is by design.");
- if (!f.delete()) {
- LOG.error("Unable to delete the file: {}", fullPath);
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args);
- }
- }
-
- // f.createNewFile();
- FileOutputStream fsStream = new FileOutputStream(f);
- inProgressObjects.put(fsStream, fullPath);
-
- return fsStream;
- } catch (IOException e) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * commit keys moves an In progress object into the metadata store so that key
- * is visible in the metadata operations from that point onwards.
- *
- * @param args Object args
- * @throws OzoneException
- */
- public void commitKey(KeyArgs args, OutputStream stream)
- throws OzoneException {
- SimpleDateFormat format =
- new SimpleDateFormat("EEE, dd MMM yyyy HH:mm:ss ZZZ", Locale.US);
- lock.writeLock().lock();
-
- try {
- byte[] bucketInfo = metadataDB.get(args.getParentName()
- .getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args);
- }
- BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
- bInfo.setKeyCount(bInfo.getKeyCount() + 1);
-
- String fileNameHash = inProgressObjects.get(stream);
- inProgressObjects.remove(stream);
- if (fileNameHash == null) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args);
- }
-
- ListKeys keyList;
- byte[] bucketListBytes = userDB.get(args.getParentName()
- .getBytes(encoding));
- keyList = ListKeys.parse(new String(bucketListBytes, encoding));
- KeyInfo keyInfo;
-
- byte[] objectBytes = metadataDB.get(args.getResourceName()
- .getBytes(encoding));
-
- if (objectBytes != null) {
- // we are overwriting an existing object.
- // TODO : Emit info for Accounting
- keyInfo = KeyInfo.parse(new String(objectBytes, encoding));
- keyList.getKeyList().remove(keyInfo);
- } else {
- keyInfo = new KeyInfo();
- }
-
- keyInfo.setCreatedOn(format.format(new Date(System.currentTimeMillis())));
-
- // TODO : support version, we need to check if versioning
- // is switched on the bucket and make appropriate calls.
- keyInfo.setVersion(0);
-
- keyInfo.setDataFileName(fileNameHash);
- keyInfo.setKeyName(args.getKeyName());
- keyInfo.setMd5hash(args.getHash());
- keyInfo.setSize(args.getSize());
-
- keyList.getKeyList().add(keyInfo);
-
- // if the key exists, we overwrite happily :). since the
- // earlier call - createObject - has overwritten the data.
-
- metadataDB.put(args.getResourceName().getBytes(encoding),
- keyInfo.toDBString().getBytes(encoding));
-
- metadataDB.put(args.getParentName().getBytes(encoding),
- bInfo.toDBString().getBytes(encoding));
-
- userDB.put(args.getParentName().getBytes(encoding),
- keyList.toDBString().getBytes(encoding));
-
- } catch (IOException e) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * deletes an key from a given bucket.
- *
- * @param args - ObjectArgs
- * @throws OzoneException
- */
- public void deleteKey(KeyArgs args) throws OzoneException {
- lock.writeLock().lock();
- try {
- byte[] bucketInfo = metadataDB.get(args.getParentName()
- .getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
- BucketInfo bInfo = BucketInfo.parse(new String(bucketInfo, encoding));
- bInfo.setKeyCount(bInfo.getKeyCount() - 1);
-
-
- byte[] bucketListBytes = userDB.get(args.getParentName()
- .getBytes(encoding));
- if (bucketListBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
- ListKeys keyList = ListKeys.parse(new String(bucketListBytes, encoding));
-
-
- byte[] objectBytes = metadataDB.get(args.getResourceName()
- .getBytes(encoding));
- if (objectBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
- }
-
- KeyInfo oInfo = KeyInfo.parse(new String(objectBytes, encoding));
- keyList.getKeyList().remove(oInfo);
-
- String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
-
- String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
- File f = new File(fullPath);
-
- if (f.exists()) {
- if (!f.delete()) {
- throw ErrorTable.newError(ErrorTable.KEY_OPERATION_CONFLICT, args);
- }
- } else {
- throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
- }
-
-
- metadataDB.delete(args.getResourceName().getBytes(encoding));
- metadataDB.put(args.getParentName().getBytes(encoding),
- bInfo.toDBString().getBytes(encoding));
- userDB.put(args.getParentName().getBytes(encoding),
- keyList.toDBString().getBytes(encoding));
- } catch (IOException e) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
- } finally {
- lock.writeLock().unlock();
- }
- }
-
- /**
- * Returns a Stream for the file.
- *
- * @param args - Object args
- * @return Stream
- * @throws IOException
- * @throws OzoneException
- */
- public LengthInputStream newKeyReader(KeyArgs args)
- throws IOException, OzoneException {
- lock.readLock().lock();
- try {
- String fileNameHash = DigestUtils.sha256Hex(args.getResourceName());
- String fullPath = storageRoot + OBJECT_DIR + fileNameHash;
- File f = new File(fullPath);
- if (!f.exists()) {
- throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, args);
- }
- long size = f.length();
-
- FileInputStream fileStream = new FileInputStream(f);
- return new LengthInputStream(fileStream, size);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Returns keys in a bucket.
- *
- * @param args
- * @return List of keys.
- * @throws IOException
- * @throws OzoneException
- */
- public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
- lock.readLock().lock();
- // TODO : Support Prefix and PrevKey lookup.
- try {
- Preconditions.checkState(args.getArgs() instanceof BucketArgs);
- BucketArgs bArgs = (BucketArgs) args.getArgs();
- byte[] bucketInfo = metadataDB.get(bArgs.getResourceName()
- .getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs);
- }
-
- byte[] bucketListBytes = userDB.get(bArgs.getResourceName()
- .getBytes(encoding));
- if (bucketListBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_RESOURCE_NAME, bArgs);
- }
- return ListKeys.parse(new String(bucketListBytes, encoding));
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * Get the Key information for a given key.
- *
- * @param args - Key Args
- * @return KeyInfo - Key Information
- * @throws OzoneException
- */
- public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
- lock.readLock().lock();
- try {
- byte[] bucketInfo = metadataDB
- .get(args.getParentName().getBytes(encoding));
- if (bucketInfo == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- byte[] bucketListBytes = userDB
- .get(args.getParentName().getBytes(encoding));
- if (bucketListBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_BUCKET_NAME, args);
- }
-
- byte[] objectBytes = metadataDB
- .get(args.getResourceName().getBytes(encoding));
- if (objectBytes == null) {
- throw ErrorTable.newError(ErrorTable.INVALID_KEY, args);
- }
-
- return KeyInfo.parse(new String(objectBytes, encoding));
- } catch (IOException e) {
- throw ErrorTable.newError(ErrorTable.SERVER_ERROR, args, e);
- } finally {
- lock.readLock().unlock();
- }
- }
-
- /**
- * This is used in updates to volume metadata.
- */
- public enum VolumeProperty {
- OWNER, QUOTA
- }
-
- /**
- * Bucket Properties.
- */
- public enum BucketProperty {
- ACLS, STORAGETYPE, VERSIONING
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
deleted file mode 100644
index 6bf6643..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.localstorage;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
deleted file mode 100644
index 6db49d5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/LengthInputStreamMessageBodyWriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.messages;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.ext.MessageBodyWriter;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-
-/**
- * Writes outbound HTTP response object bytes. The content length is determined
- * from the {@link LengthInputStream}.
- */
-public final class LengthInputStreamMessageBodyWriter
- implements MessageBodyWriter<LengthInputStream> {
- private static final int CHUNK_SIZE = 8192;
-
- @Override
- public long getSize(LengthInputStream lis, Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType) {
- return lis.getLength();
- }
-
- @Override
- public boolean isWriteable(Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType) {
- return LengthInputStream.class.isAssignableFrom(type);
- }
-
- @Override
- public void writeTo(LengthInputStream lis, Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType,
- MultivaluedMap<String, Object> httpHeaders,
- OutputStream out) throws IOException {
- IOUtils.copyBytes(lis, out, CHUNK_SIZE);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
deleted file mode 100644
index ad637af..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/StringMessageBodyWriter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.messages;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.MultivaluedMap;
-import javax.ws.rs.ext.MessageBodyWriter;
-import java.io.ByteArrayInputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.Type;
-
-/**
- * Writes outbound HTTP response strings. We use this rather than the built-in
- * writer so that we can determine content length from the string length instead
- * of possibly falling back to a chunked response.
- */
-public final class StringMessageBodyWriter implements
- MessageBodyWriter<String> {
- private static final int CHUNK_SIZE = 8192;
-
- @Override
- public long getSize(String str, Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType) {
- return str.length();
- }
-
- @Override
- public boolean isWriteable(Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType) {
- return String.class.isAssignableFrom(type);
- }
-
- @Override
- public void writeTo(String str, Class<?> type, Type genericType,
- Annotation[] annotations, MediaType mediaType,
- MultivaluedMap<String, Object> httpHeaders,
- OutputStream out) throws IOException {
- IOUtils.copyBytes(new ByteArrayInputStream(
- str.getBytes(OzoneUtils.ENCODING)), out, CHUNK_SIZE);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
deleted file mode 100644
index 273b3f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/messages/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.messages;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
deleted file mode 100644
index 3d9db20..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/CloseableCleanupListener.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.Closeable;
-
-/**
- * A {@link ChannelFutureListener} that closes {@link Closeable} resources.
- */
-final class CloseableCleanupListener implements ChannelFutureListener {
-
- private final Closeable[] closeables;
-
- /**
- * Creates a new CloseableCleanupListener.
- *
- * @param closeables any number of closeable resources
- */
- CloseableCleanupListener(Closeable... closeables) {
- this.closeables = closeables;
- }
-
- @Override
- public void operationComplete(ChannelFuture future) {
- IOUtils.cleanupWithLogger(null, closeables);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
deleted file mode 100644
index 89c196c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpResponseStatus.INTERNAL_SERVER_ERROR;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-/**
- * Abstract base class for the multiple Netty channel handlers used in the
- * Object Store Netty channel pipeline.
- */
-abstract class ObjectStoreChannelHandler<T>
- extends SimpleChannelInboundHandler<T> {
-
- /** Log usable in all subclasses. */
- protected static final Logger LOG =
- LoggerFactory.getLogger(ObjectStoreChannelHandler.class);
-
- /**
- * Handles uncaught exceptions in the channel pipeline by sending an internal
- * server error response if the channel is still active.
- *
- * @param ctx ChannelHandlerContext to receive response
- * @param cause Throwable that was unhandled in the channel pipeline
- */
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- LOG.error("Unexpected exception in Netty pipeline.", cause);
- if (ctx.channel().isActive()) {
- sendErrorResponse(ctx, INTERNAL_SERVER_ERROR);
- }
- }
-
- /**
- * Sends an error response. This method is used when an unexpected error is
- * encountered within the channel pipeline, outside of the actual Object Store
- * application. It always closes the connection, because we can't in general
- * know the state of the connection when these errors occur, so attempting to
- * keep the connection alive could be unpredictable.
- *
- * @param ctx ChannelHandlerContext to receive response
- * @param status HTTP response status
- */
- protected static void sendErrorResponse(ChannelHandlerContext ctx,
- HttpResponseStatus status) {
- HttpResponse nettyResp = new DefaultFullHttpResponse(HTTP_1_1, status);
- nettyResp.headers().set(CONTENT_LENGTH, 0);
- nettyResp.headers().set(CONNECTION, CLOSE);
- ctx.writeAndFlush(nettyResp).addListener(ChannelFutureListener.CLOSE);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
deleted file mode 100644
index c7b516f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainer.java
+++ /dev/null
@@ -1,348 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONTENT_LENGTH;
-import static io.netty.handler.codec.http.HttpHeaders.Names.CONNECTION;
-import static io.netty.handler.codec.http.HttpHeaders.Names.TRANSFER_ENCODING;
-import static io.netty.handler.codec.http.HttpHeaders.Names.HOST;
-import static io.netty.handler.codec.http.HttpHeaders.Values.KEEP_ALIVE;
-import static io.netty.handler.codec.http.HttpHeaders.Values.CLOSE;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.TimeUnit;
-
-import com.sun.jersey.core.header.InBoundHeaders;
-import com.sun.jersey.spi.container.ContainerRequest;
-import com.sun.jersey.spi.container.ContainerResponse;
-import com.sun.jersey.spi.container.ContainerResponseWriter;
-import com.sun.jersey.spi.container.WebApplication;
-
-import io.netty.handler.codec.http.DefaultHttpResponse;
-//import io.netty.handler.codec.http.HttpUtil;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.HttpResponseStatus;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.handlers.StorageHandlerBuilder;
-
-/**
- * This is a custom Jersey container that hosts the Object Store web
- * application. It supports dispatching an inbound Netty {@link HttpRequest}
- * to the Object Store Jersey application. Request dispatching must run
- * asynchronously, because the Jersey application must consume the inbound
- * HTTP request from a piped stream and produce the outbound HTTP response
- * for another piped stream.The Netty channel handlers consume the connected
- * ends of these piped streams. Request dispatching cannot run directly on
- * the Netty threads, or there would be a risk of deadlock (one thread
- * producing/consuming its end of the pipe while no other thread is
- * producing/consuming the opposite end).
- */
-public final class ObjectStoreJerseyContainer {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ObjectStoreJerseyContainer.class);
-
- private final WebApplication webapp;
-
- private StorageHandler storageHandler;
-
- /**
- * Creates a new ObjectStoreJerseyContainer.
- *
- * @param webapp web application
- */
- public ObjectStoreJerseyContainer(WebApplication webapp) {
- this.webapp = webapp;
- }
-
- /**
- * Sets the {@link StorageHandler}. This must be called before dispatching any
- * requests.
- *
- * @param newStorageHandler {@link StorageHandler} implementation
- */
- public void setStorageHandler(StorageHandler newStorageHandler) {
- this.storageHandler = newStorageHandler;
- }
-
- /**
- * Asynchronously executes an HTTP request.
- *
- * @param nettyReq HTTP request
- * @param reqIn input stream for reading request body
- * @param respOut output stream for writing response body
- */
- public Future<HttpResponse> dispatch(HttpRequest nettyReq, InputStream reqIn,
- OutputStream respOut) {
- // The request executes on a separate background thread. As soon as enough
- // processing has completed to bootstrap the outbound response, the thread
- // counts down on a latch. This latch also unblocks callers trying to get
- // the asynchronous response out of the returned future.
- final CountDownLatch latch = new CountDownLatch(1);
- final RequestRunner runner = new RequestRunner(nettyReq, reqIn, respOut,
- latch);
- final Thread thread = new Thread(runner);
- thread.setDaemon(true);
- thread.start();
- return new Future<HttpResponse>() {
-
- private volatile boolean isCancelled = false;
-
- @Override
- public boolean cancel(boolean mayInterruptIfRunning) {
- if (latch.getCount() == 0) {
- return false;
- }
- if (!mayInterruptIfRunning) {
- return false;
- }
- if (!thread.isAlive()) {
- return false;
- }
- thread.interrupt();
- try {
- thread.join();
- } catch (InterruptedException e) {
- LOG.info("Interrupted while attempting to cancel dispatch thread.");
- Thread.currentThread().interrupt();
- return false;
- }
- isCancelled = true;
- return true;
- }
-
- @Override
- public HttpResponse get()
- throws InterruptedException, ExecutionException {
- checkCancelled();
- latch.await();
- return this.getOrThrow();
- }
-
- @Override
- public HttpResponse get(long timeout, TimeUnit unit)
- throws InterruptedException, ExecutionException, TimeoutException {
- checkCancelled();
- if (!latch.await(timeout, unit)) {
- throw new TimeoutException(String.format(
- "Timed out waiting for HttpResponse after %d %s.",
- timeout, unit.toString().toLowerCase()));
- }
- return this.getOrThrow();
- }
-
- @Override
- public boolean isCancelled() {
- return isCancelled;
- }
-
- @Override
- public boolean isDone() {
- return !isCancelled && latch.getCount() == 0;
- }
-
- private void checkCancelled() {
- if (isCancelled()) {
- throw new CancellationException();
- }
- }
-
- private HttpResponse getOrThrow() throws ExecutionException {
- try {
- return runner.getResponse();
- } catch (Exception e) {
- throw new ExecutionException(e);
- }
- }
- };
- }
-
- /**
- * Runs the actual handling of the HTTP request.
- */
- private final class RequestRunner implements Runnable,
- ContainerResponseWriter {
-
- private final CountDownLatch latch;
- private final HttpRequest nettyReq;
- private final InputStream reqIn;
- private final OutputStream respOut;
-
- private Exception exception;
- private HttpResponse nettyResp;
-
- /**
- * Creates a new RequestRunner.
- *
- * @param nettyReq HTTP request
- * @param reqIn input stream for reading request body
- * @param respOut output stream for writing response body
- * @param latch for coordinating asynchronous return of HTTP response
- */
- RequestRunner(HttpRequest nettyReq, InputStream reqIn,
- OutputStream respOut, CountDownLatch latch) {
- this.latch = latch;
- this.nettyReq = nettyReq;
- this.reqIn = reqIn;
- this.respOut = respOut;
- }
-
- @Override
- public void run() {
- LOG.trace("begin RequestRunner, nettyReq = {}", this.nettyReq);
- StorageHandlerBuilder.setStorageHandler(
- ObjectStoreJerseyContainer.this.storageHandler);
- try {
- ContainerRequest jerseyReq = nettyRequestToJerseyRequest(
- ObjectStoreJerseyContainer.this.webapp, this.nettyReq, this.reqIn);
- ObjectStoreJerseyContainer.this.webapp.handleRequest(jerseyReq, this);
- } catch (Exception e) {
- LOG.error("Error running Jersey Request Runner", e);
- this.exception = e;
- this.latch.countDown();
- } finally {
- IOUtils.cleanupWithLogger(null, this.reqIn, this.respOut);
- StorageHandlerBuilder.removeStorageHandler();
- }
- LOG.trace("end RequestRunner, nettyReq = {}", this.nettyReq);
- }
-
- /**
- * This is a callback triggered by Jersey as soon as dispatch has completed
- * to the point of knowing what kind of response to return. We save the
- * response and trigger the latch to unblock callers waiting on the
- * asynchronous return of the response. Our response always sets a
- * Content-Length header. (We do not support Transfer-Encoding: chunked.)
- * We also return the output stream for Jersey to use for writing the
- * response body.
- *
- * @param contentLength length of response
- * @param jerseyResp HTTP response returned by Jersey
- * @return OutputStream for Jersey to use for writing the response body
- */
- @Override
- public OutputStream writeStatusAndHeaders(long contentLength,
- ContainerResponse jerseyResp) {
- LOG.trace(
- "begin writeStatusAndHeaders, contentLength = {}, jerseyResp = {}.",
- contentLength, jerseyResp);
- this.nettyResp = jerseyResponseToNettyResponse(jerseyResp);
- this.nettyResp.headers().set(CONTENT_LENGTH, Math.max(0, contentLength));
- this.nettyResp.headers().set(CONNECTION,
- HttpHeaders.isKeepAlive(this.nettyReq) ? KEEP_ALIVE : CLOSE);
- this.latch.countDown();
- LOG.trace(
- "end writeStatusAndHeaders, contentLength = {}, jerseyResp = {}.",
- contentLength, jerseyResp);
- return this.respOut;
- }
-
- /**
- * This is a callback triggered by Jersey after it has completed writing the
- * response body to the stream. We must close the stream here to unblock
- * the Netty thread consuming the last chunk of the response from the input
- * end of the piped stream.
- *
- * @throws IOException if there is an I/O error
- */
- @Override
- public void finish() throws IOException {
- IOUtils.cleanupWithLogger(null, this.respOut);
- }
-
- /**
- * Gets the HTTP response calculated by the Jersey application, or throws an
- * exception if an error occurred during processing. It only makes sense to
- * call this method after waiting on the latch to trigger.
- *
- * @return HTTP response
- * @throws Exception if there was an error executing the request
- */
- public HttpResponse getResponse() throws Exception {
- if (this.exception != null) {
- throw this.exception;
- }
- return this.nettyResp;
- }
- }
-
- /**
- * Converts a Jersey HTTP response object to a Netty HTTP response object.
- *
- * @param jerseyResp Jersey HTTP response
- * @return Netty HTTP response
- */
- private static HttpResponse jerseyResponseToNettyResponse(
- ContainerResponse jerseyResp) {
- HttpResponse nettyResp = new DefaultHttpResponse(HTTP_1_1,
- HttpResponseStatus.valueOf(jerseyResp.getStatus()));
- for (Map.Entry<String, List<Object>> header :
- jerseyResp.getHttpHeaders().entrySet()) {
- if (!header.getKey().equalsIgnoreCase(CONTENT_LENGTH.toString()) &&
- !header.getKey().equalsIgnoreCase(TRANSFER_ENCODING.toString())) {
- nettyResp.headers().set(header.getKey(), header.getValue());
- }
- }
- return nettyResp;
- }
-
- /**
- * Converts a Netty HTTP request object to a Jersey HTTP request object.
- *
- * @param webapp web application
- * @param nettyReq Netty HTTP request
- * @param reqIn input stream for reading request body
- * @return Jersey HTTP request
- * @throws URISyntaxException if there is an error handling the request URI
- */
- private static ContainerRequest nettyRequestToJerseyRequest(
- WebApplication webapp, HttpRequest nettyReq, InputStream reqIn)
- throws URISyntaxException {
- HttpHeaders nettyHeaders = nettyReq.headers();
- InBoundHeaders jerseyHeaders = new InBoundHeaders();
- for (String name : nettyHeaders.names()) {
- jerseyHeaders.put(name, nettyHeaders.getAll(name));
- }
- String host = nettyHeaders.get(HOST);
- String scheme = host.startsWith("https") ? "https://" : "http://";
- String baseUri = scheme + host + "/";
- String reqUri = scheme + host + nettyReq.getUri();
- LOG.trace("baseUri = {}, reqUri = {}", baseUri, reqUri);
- return new ContainerRequest(webapp, nettyReq.getMethod().name(),
- new URI(baseUri), new URI(reqUri), jerseyHeaders, reqIn);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
deleted file mode 100644
index e943969..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/ObjectStoreJerseyContainerProvider.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import com.sun.jersey.api.container.ContainerException;
-import com.sun.jersey.api.core.ResourceConfig;
-import com.sun.jersey.spi.container.ContainerProvider;
-import com.sun.jersey.spi.container.WebApplication;
-
-/**
- * This is a Jersey {@link ContainerProvider} capable of boostrapping the
- * Object Store web application into a custom container. It must be registered
- * using the Java service loader mechanism by listing it in
- * META-INF/services/com.sun.jersey.spi.container.ContainerProvider .
- */
-public final class ObjectStoreJerseyContainerProvider
- implements ContainerProvider<ObjectStoreJerseyContainer> {
-
- @Override
- public ObjectStoreJerseyContainer createContainer(
- Class<ObjectStoreJerseyContainer> type, ResourceConfig conf,
- WebApplication webapp) throws ContainerException {
- return new ObjectStoreJerseyContainer(webapp);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
deleted file mode 100644
index 0a2f22d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestContentObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,117 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelFuture;
-import io.netty.channel.ChannelFutureListener;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.HttpContent;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import io.netty.handler.codec.http.LastHttpContent;
-import io.netty.handler.stream.ChunkedStream;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.concurrent.Future;
-
-/**
- * Object Store Netty channel pipeline handler that handles inbound
- * {@link HttpContent} fragments for the request body by sending the bytes into
- * the pipe so that the application dispatch thread can read it.
- * After receiving the {@link LastHttpContent}, this handler also flushes the
- * response.
- */
-public final class RequestContentObjectStoreChannelHandler
- extends ObjectStoreChannelHandler<HttpContent> {
-
- private final HttpRequest nettyReq;
- private final Future<HttpResponse> nettyResp;
- private final OutputStream reqOut;
- private final InputStream respIn;
- private ObjectStoreJerseyContainer jerseyContainer;
-
- /**
- * Creates a new RequestContentObjectStoreChannelHandler.
- *
- * @param nettyReq HTTP request
- * @param nettyResp asynchronous HTTP response
- * @param reqOut output stream for writing request body
- * @param respIn input stream for reading response body
- * @param jerseyContainer jerseyContainer to handle the request
- */
- public RequestContentObjectStoreChannelHandler(HttpRequest nettyReq,
- Future<HttpResponse> nettyResp, OutputStream reqOut, InputStream respIn,
- ObjectStoreJerseyContainer jerseyContainer) {
- this.nettyReq = nettyReq;
- this.nettyResp = nettyResp;
- this.reqOut = reqOut;
- this.respIn = respIn;
- this.jerseyContainer = jerseyContainer;
- }
-
- @Override
- public void channelReadComplete(ChannelHandlerContext ctx) throws Exception {
- ctx.flush();
- }
-
- @Override
- public void channelRead0(ChannelHandlerContext ctx, HttpContent content)
- throws Exception {
- LOG.trace(
- "begin RequestContentObjectStoreChannelHandler channelRead0, " +
- "ctx = {}, content = {}", ctx, content);
- content.content().readBytes(this.reqOut, content.content().readableBytes());
- if (content instanceof LastHttpContent) {
- IOUtils.cleanupWithLogger(null, this.reqOut);
- ctx.write(this.nettyResp.get());
- ChannelFuture respFuture = ctx.writeAndFlush(new ChunkedStream(
- this.respIn));
- respFuture.addListener(new CloseableCleanupListener(this.respIn));
- if (!HttpHeaders.isKeepAlive(this.nettyReq)) {
- respFuture.addListener(ChannelFutureListener.CLOSE);
- } else {
- respFuture.addListener(new ChannelFutureListener() {
- @Override
- public void operationComplete(ChannelFuture future) throws Exception {
- // Notify client this is the last content for current request.
- ctx.writeAndFlush(LastHttpContent.EMPTY_LAST_CONTENT);
- // Reset the pipeline handler for next request to reuses the
- // same connection.
- RequestDispatchObjectStoreChannelHandler h =
- new RequestDispatchObjectStoreChannelHandler(jerseyContainer);
- ctx.pipeline().replace(ctx.pipeline().last(),
- RequestDispatchObjectStoreChannelHandler.class.getSimpleName(),
- h);
- }
- });
- }
- }
- LOG.trace(
- "end RequestContentObjectStoreChannelHandler channelRead0, " +
- "ctx = {}, content = {}", ctx, content);
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- super.exceptionCaught(ctx, cause);
- IOUtils.cleanupWithLogger(null, this.reqOut, this.respIn);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
deleted file mode 100644
index add827a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/RequestDispatchObjectStoreChannelHandler.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.netty;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.http.DefaultFullHttpResponse;
-import io.netty.handler.codec.http.HttpHeaders;
-import io.netty.handler.codec.http.HttpRequest;
-import io.netty.handler.codec.http.HttpResponse;
-import org.apache.hadoop.io.IOUtils;
-
-import java.io.PipedInputStream;
-import java.io.PipedOutputStream;
-import java.util.concurrent.Future;
-
-import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST;
-import static io.netty.handler.codec.http.HttpResponseStatus.CONTINUE;
-import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1;
-
-/**
- * Object Store Netty channel pipeline handler that handles an inbound
- * {@link HttpRequest} by dispatching it to the Object Store Jersey container.
- * The handler establishes 2 sets of connected piped streams: one for inbound
- * request handling and another for outbound response handling. The relevant
- * ends of these pipes are handed off to the Jersey application dispatch and the
- * next channel handler, which is responsible for streaming in the inbound
- * request body and flushing out the response body.
- */
-public final class RequestDispatchObjectStoreChannelHandler
- extends ObjectStoreChannelHandler<HttpRequest> {
-
- private final ObjectStoreJerseyContainer jerseyContainer;
-
- private PipedInputStream reqIn;
- private PipedOutputStream reqOut;
- private PipedInputStream respIn;
- private PipedOutputStream respOut;
-
- /**
- * Creates a new RequestDispatchObjectStoreChannelHandler.
- *
- * @param jerseyContainer Object Store application Jersey container for
- * request dispatch
- */
- public RequestDispatchObjectStoreChannelHandler(
- ObjectStoreJerseyContainer jerseyContainer) {
- this.jerseyContainer = jerseyContainer;
- }
-
- @Override
- public void channelRead0(ChannelHandlerContext ctx, HttpRequest nettyReq)
- throws Exception {
- LOG.trace("begin RequestDispatchObjectStoreChannelHandler channelRead0, " +
- "ctx = {}, nettyReq = {}", ctx, nettyReq);
- if (!nettyReq.getDecoderResult().isSuccess()) {
- sendErrorResponse(ctx, BAD_REQUEST);
- return;
- }
-
- this.reqIn = new PipedInputStream();
- this.reqOut = new PipedOutputStream(reqIn);
- this.respIn = new PipedInputStream();
- this.respOut = new PipedOutputStream(respIn);
-
- if (HttpHeaders.is100ContinueExpected(nettyReq)) {
- LOG.trace("Sending continue response.");
- ctx.writeAndFlush(new DefaultFullHttpResponse(HTTP_1_1, CONTINUE));
- }
-
- Future<HttpResponse> nettyResp = this.jerseyContainer.dispatch(nettyReq,
- reqIn, respOut);
-
- ctx.pipeline().replace(this,
- RequestContentObjectStoreChannelHandler.class.getSimpleName(),
- new RequestContentObjectStoreChannelHandler(nettyReq, nettyResp,
- reqOut, respIn, jerseyContainer));
-
- LOG.trace("end RequestDispatchObjectStoreChannelHandler channelRead0, " +
- "ctx = {}, nettyReq = {}", ctx, nettyReq);
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- super.exceptionCaught(ctx, cause);
- IOUtils.cleanupWithLogger(null, this.reqIn, this.reqOut, this.respIn,
- this.respOut);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
deleted file mode 100644
index f4aa675..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/netty/package-info.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.web.netty;
-
-import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
deleted file mode 100644
index 015d46e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Handler.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClient;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.http.client.utils.URIBuilder;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Common interface for command handling.
- */
-public abstract class Handler {
- protected OzoneRestClient client;
-
- /**
- * Constructs a client object.
- */
- public Handler() {
- client = new OzoneRestClient();
- }
-
- /**
- * Executes the Client command.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- protected abstract void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException;
-
- /**
- * verifies user provided URI.
- *
- * @param uri - UriString
- * @return URI
- * @throws URISyntaxException
- * @throws OzoneException
- */
- protected URI verifyURI(String uri) throws URISyntaxException,
- OzoneException {
- if ((uri == null) || uri.isEmpty()) {
- throw new OzoneRestClientException(
- "Ozone URI is needed to execute this command.");
- }
- URIBuilder ozoneURI = new URIBuilder(uri);
-
- if (ozoneURI.getPort() == 0) {
- ozoneURI.setPort(Shell.DEFAULT_OZONE_PORT);
- }
- return ozoneURI.build();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
deleted file mode 100644
index c5fd749..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLogImpl.java
+++ /dev/null
@@ -1,351 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.DELETED_BLOCK_DB;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-
-/**
- * A implement class of {@link DeletedBlockLog}, and it uses
- * K/V db to maintain block deletion transactions between scm and datanode.
- * This is a very basic implementation, it simply scans the log and
- * memorize the position that scanned by last time, and uses this to
- * determine where the next scan starts. It has no notion about weight
- * of each transaction so as long as transaction is still valid, they get
- * equally same chance to be retrieved which only depends on the nature
- * order of the transaction ID.
- */
-public class DeletedBlockLogImpl implements DeletedBlockLog {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(DeletedBlockLogImpl.class);
-
- private static final byte[] LATEST_TXID =
- DFSUtil.string2Bytes("#LATEST_TXID#");
-
- private final int maxRetry;
- private final MetadataStore deletedStore;
- private final Lock lock;
- // The latest id of deleted blocks in the db.
- private long lastTxID;
- private long lastReadTxID;
-
- public DeletedBlockLogImpl(Configuration conf) throws IOException {
- maxRetry = conf.getInt(OZONE_SCM_BLOCK_DELETION_MAX_RETRY,
- OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT);
-
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
- File deletedLogDbPath = new File(scmMetaDataDir, DELETED_BLOCK_DB);
- int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
- // Load store of all transactions.
- deletedStore = MetadataStoreBuilder.newBuilder()
- .setCreateIfMissing(true)
- .setConf(conf)
- .setDbFile(deletedLogDbPath)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
-
- this.lock = new ReentrantLock();
- // start from the head of deleted store.
- lastReadTxID = 0;
- lastTxID = findLatestTxIDInStore();
- }
-
- @VisibleForTesting
- MetadataStore getDeletedStore() {
- return deletedStore;
- }
-
- /**
- * There is no need to lock before reading because
- * it's only used in construct method.
- *
- * @return latest txid.
- * @throws IOException
- */
- private long findLatestTxIDInStore() throws IOException {
- long txid = 0;
- byte[] value = deletedStore.get(LATEST_TXID);
- if (value != null) {
- txid = Longs.fromByteArray(value);
- }
- return txid;
- }
-
- @Override
- public List<DeletedBlocksTransaction> getTransactions(
- int count) throws IOException {
- List<DeletedBlocksTransaction> result = new ArrayList<>();
- MetadataKeyFilter getNextTxID = (preKey, currentKey, nextKey)
- -> Longs.fromByteArray(currentKey) > lastReadTxID;
- MetadataKeyFilter avoidInvalidTxid = (preKey, currentKey, nextKey)
- -> !Arrays.equals(LATEST_TXID, currentKey);
- lock.lock();
- try {
- deletedStore.iterate(null, (key, value) -> {
- if (getNextTxID.filterKey(null, key, null) &&
- avoidInvalidTxid.filterKey(null, key, null)) {
- DeletedBlocksTransaction block = DeletedBlocksTransaction
- .parseFrom(value);
- if (block.getCount() > -1 && block.getCount() <= maxRetry) {
- result.add(block);
- }
- }
- return result.size() < count;
- });
- // Scan the metadata from the beginning.
- if (result.size() < count || result.size() < 1) {
- lastReadTxID = 0;
- } else {
- lastReadTxID = result.get(result.size() - 1).getTxID();
- }
- } finally {
- lock.unlock();
- }
- return result;
- }
-
- @Override
- public List<DeletedBlocksTransaction> getFailedTransactions()
- throws IOException {
- lock.lock();
- try {
- final List<DeletedBlocksTransaction> failedTXs = Lists.newArrayList();
- deletedStore.iterate(null, (key, value) -> {
- if (!Arrays.equals(LATEST_TXID, key)) {
- DeletedBlocksTransaction delTX =
- DeletedBlocksTransaction.parseFrom(value);
- if (delTX.getCount() == -1) {
- failedTXs.add(delTX);
- }
- }
- return true;
- });
- return failedTXs;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- *
- * @param txIDs - transaction ID.
- * @throws IOException
- */
- @Override
- public void incrementCount(List<Long> txIDs) throws IOException {
- BatchOperation batch = new BatchOperation();
- lock.lock();
- try {
- for(Long txID : txIDs) {
- try {
- DeletedBlocksTransaction block = DeletedBlocksTransaction
- .parseFrom(deletedStore.get(Longs.toByteArray(txID)));
- DeletedBlocksTransaction.Builder builder = block.toBuilder();
- int currentCount = block.getCount();
- if (currentCount > -1) {
- builder.setCount(++currentCount);
- }
- // if the retry time exceeds the maxRetry value
- // then set the retry value to -1, stop retrying, admins can
- // analyze those blocks and purge them manually by SCMCli.
- if (currentCount > maxRetry) {
- builder.setCount(-1);
- }
- deletedStore.put(Longs.toByteArray(txID),
- builder.build().toByteArray());
- } catch (IOException ex) {
- LOG.warn("Cannot increase count for txID " + txID, ex);
- }
- }
- deletedStore.writeBatch(batch);
- } finally {
- lock.unlock();
- }
- }
-
- private DeletedBlocksTransaction constructNewTransaction(long txID,
- String containerName, List<String> blocks) {
- return DeletedBlocksTransaction.newBuilder()
- .setTxID(txID)
- .setContainerName(containerName)
- .addAllBlockID(blocks)
- .setCount(0)
- .build();
- }
-
- /**
- * {@inheritDoc}
- *
- * @param txIDs - transaction IDs.
- * @throws IOException
- */
- @Override
- public void commitTransactions(List<Long> txIDs) throws IOException {
- lock.lock();
- try {
- for (Long txID : txIDs) {
- try {
- deletedStore.delete(Longs.toByteArray(txID));
- } catch (IOException ex) {
- LOG.warn("Cannot commit txID " + txID, ex);
- }
- }
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- *
- * @param containerName - container name.
- * @param blocks - blocks that belong to the same container.
- * @throws IOException
- */
- @Override
- public void addTransaction(String containerName, List<String> blocks)
- throws IOException {
- BatchOperation batch = new BatchOperation();
- lock.lock();
- try {
- DeletedBlocksTransaction tx = constructNewTransaction(lastTxID + 1,
- containerName, blocks);
- byte[] key = Longs.toByteArray(lastTxID + 1);
-
- batch.put(key, tx.toByteArray());
- batch.put(LATEST_TXID, Longs.toByteArray(lastTxID + 1));
-
- deletedStore.writeBatch(batch);
- lastTxID += 1;
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public int getNumOfValidTransactions() throws IOException {
- lock.lock();
- try {
- final AtomicInteger num = new AtomicInteger(0);
- deletedStore.iterate(null, (key, value) -> {
- // Exclude latest txid record
- if (!Arrays.equals(LATEST_TXID, key)) {
- DeletedBlocksTransaction delTX =
- DeletedBlocksTransaction.parseFrom(value);
- if (delTX.getCount() > -1) {
- num.incrementAndGet();
- }
- }
- return true;
- });
- return num.get();
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- *
- * @param containerBlocksMap a map of containerBlocks.
- * @throws IOException
- */
- @Override
- public void addTransactions(Map<String, List<String>> containerBlocksMap)
- throws IOException {
- BatchOperation batch = new BatchOperation();
- lock.lock();
- try {
- long currentLatestID = lastTxID;
- for (Map.Entry<String, List<String>> entry :
- containerBlocksMap.entrySet()) {
- currentLatestID += 1;
- byte[] key = Longs.toByteArray(currentLatestID);
- DeletedBlocksTransaction tx = constructNewTransaction(currentLatestID,
- entry.getKey(), entry.getValue());
- batch.put(key, tx.toByteArray());
- }
- lastTxID = currentLatestID;
- batch.put(LATEST_TXID, Longs.toByteArray(lastTxID));
- deletedStore.writeBatch(batch);
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public void close() throws IOException {
- if (deletedStore != null) {
- deletedStore.close();
- }
- }
-
- @Override
- public void getTransactions(DatanodeDeletedBlockTransactions transactions)
- throws IOException {
- lock.lock();
- try {
- deletedStore.iterate(null, (key, value) -> {
- if (!Arrays.equals(LATEST_TXID, key)) {
- DeletedBlocksTransaction block = DeletedBlocksTransaction
- .parseFrom(value);
-
- if (block.getCount() > -1 && block.getCount() <= maxRetry) {
- transactions.addTransaction(block);
- }
- return !transactions.isFull();
- }
- return true;
- });
- } finally {
- lock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
deleted file mode 100644
index eeef14b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/SCMBlockDeletingService.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult.EmptyTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * A background service running in SCM to delete blocks. This service scans
- * block deletion log in certain interval and caches block deletion commands
- * in {@link org.apache.hadoop.ozone.scm.node.CommandQueue}, asynchronously
- * SCM HB thread polls cached commands and sends them to datanode for physical
- * processing.
- */
-public class SCMBlockDeletingService extends BackgroundService {
-
- static final Logger LOG =
- LoggerFactory.getLogger(SCMBlockDeletingService.class);
-
- // ThreadPoolSize=2, 1 for scheduler and the other for the scanner.
- private final static int BLOCK_DELETING_SERVICE_CORE_POOL_SIZE = 2;
- private final DeletedBlockLog deletedBlockLog;
- private final Mapping mappingService;
- private final NodeManager nodeManager;
-
- // Block delete limit size is dynamically calculated based on container
- // delete limit size (ozone.block.deleting.container.limit.per.interval)
- // that configured for datanode. To ensure DN not wait for
- // delete commands, we use this value multiply by a factor 2 as the final
- // limit TX size for each node.
- // Currently we implement a throttle algorithm that throttling delete blocks
- // for each datanode. Each node is limited by the calculation size. Firstly
- // current node info is fetched from nodemanager, then scan entire delLog
- // from the beginning to end. If one node reaches maximum value, its records
- // will be skipped. If not, keep scanning until it reaches maximum value.
- // Once all node are full, the scan behavior will stop.
- private int blockDeleteLimitSize;
-
- public SCMBlockDeletingService(DeletedBlockLog deletedBlockLog,
- Mapping mapper, NodeManager nodeManager,
- long interval, long serviceTimeout, Configuration conf) {
- super("SCMBlockDeletingService", interval, TimeUnit.MILLISECONDS,
- BLOCK_DELETING_SERVICE_CORE_POOL_SIZE, serviceTimeout);
- this.deletedBlockLog = deletedBlockLog;
- this.mappingService = mapper;
- this.nodeManager = nodeManager;
-
- int containerLimit = conf.getInt(
- OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL,
- OZONE_BLOCK_DELETING_CONTAINER_LIMIT_PER_INTERVAL_DEFAULT);
- Preconditions.checkArgument(containerLimit > 0,
- "Container limit size should be " + "positive.");
- // Use container limit value multiply by a factor 2 to ensure DN
- // not wait for orders.
- this.blockDeleteLimitSize = containerLimit * 2;
- }
-
- @Override
- public BackgroundTaskQueue getTasks() {
- BackgroundTaskQueue queue = new BackgroundTaskQueue();
- queue.add(new DeletedBlockTransactionScanner());
- return queue;
- }
-
- private class DeletedBlockTransactionScanner
- implements BackgroundTask<EmptyTaskResult> {
-
- @Override
- public int getPriority() {
- return 1;
- }
-
- @Override
- public EmptyTaskResult call() throws Exception {
- int dnTxCount = 0;
- long startTime = Time.monotonicNow();
- // Scan SCM DB in HB interval and collect a throttled list of
- // to delete blocks.
- LOG.debug("Running DeletedBlockTransactionScanner");
- DatanodeDeletedBlockTransactions transactions = null;
- List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- if (datanodes != null) {
- transactions = new DatanodeDeletedBlockTransactions(mappingService,
- blockDeleteLimitSize, datanodes.size());
- try {
- deletedBlockLog.getTransactions(transactions);
- } catch (IOException e) {
- // We may tolerant a number of failures for sometime
- // but if it continues to fail, at some point we need to raise
- // an exception and probably fail the SCM ? At present, it simply
- // continues to retry the scanning.
- LOG.error("Failed to get block deletion transactions from delTX log",
- e);
- }
- LOG.debug("Scanned deleted blocks log and got {} delTX to process.",
- transactions.getTXNum());
- }
-
- if (transactions != null && !transactions.isEmpty()) {
- for (DatanodeID datanodeID : transactions.getDatanodes()) {
- List<DeletedBlocksTransaction> dnTXs = transactions
- .getDatanodeTransactions(datanodeID);
- if (dnTXs != null && !dnTXs.isEmpty()) {
- dnTxCount += dnTXs.size();
- // TODO commandQueue needs a cap.
- // We should stop caching new commands if num of un-processed
- // command is bigger than a limit, e.g 50. In case datanode goes
- // offline for sometime, the cached commands be flooded.
- nodeManager.addDatanodeCommand(datanodeID,
- new DeleteBlocksCommand(dnTXs));
- LOG.debug(
- "Added delete block command for datanode {} in the queue,"
- + " number of delete block transactions: {}, TxID list: {}",
- datanodeID, dnTXs.size(), String.join(",",
- transactions.getTransactionIDList(datanodeID)));
- }
- }
- }
-
- if (dnTxCount > 0) {
- LOG.info(
- "Totally added {} delete blocks command for"
- + " {} datanodes, task elapsed time: {}ms",
- dnTxCount, transactions.getDatanodes().size(),
- Time.monotonicNow() - startTime);
- }
-
- return EmptyTaskResult.newResult();
- }
- }
-
- @VisibleForTesting
- public void setBlockDeleteTXNum(int numTXs) {
- blockDeleteLimitSize = numTXs;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
deleted file mode 100644
index 3c56627..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-/**
- * This package contains routines to manage the block location and
- * mapping inside SCM
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
deleted file mode 100644
index 451b1b1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneBaseCLI.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.util.Tool;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * This class is the base CLI for scm, ksm and scmadm.
- */
-public abstract class OzoneBaseCLI extends Configured implements Tool {
-
- protected abstract int dispatch(CommandLine cmd, Options opts)
- throws IOException, OzoneException, URISyntaxException;
-
- protected abstract CommandLine parseArgs(String[] argv, Options opts)
- throws ParseException;
-
- protected abstract Options getOptions();
-
- protected abstract void displayHelp();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
deleted file mode 100644
index da12bd4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/OzoneCommandHandler.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-import java.io.PrintStream;
-
-/**
- * The abstract class of all SCM CLI commands.
- */
-public abstract class OzoneCommandHandler {
-
- private ScmClient scmClient;
- protected PrintStream out = System.out;
- protected PrintStream err = System.err;
-
- /**
- * Constructs a handler object.
- */
- public OzoneCommandHandler(ScmClient scmClient) {
- this.scmClient = scmClient;
- }
-
- protected ScmClient getScmClient() {
- return scmClient;
- }
-
- /**
- * Sets customized output stream to redirect the stdout to somewhere else.
- * @param out
- */
- public void setOut(PrintStream out) {
- this.out = out;
- }
-
- /**
- * Sets customized error stream to redirect the stderr to somewhere else.
- * @param err
- */
- public void setErr(PrintStream err) {
- this.err = err;
- }
-
- public void logOut(String msg, String ... variable) {
- this.out.println(String.format(msg, variable));
- }
-
- /**
- * Executes the Client command.
- *
- * @param cmd - CommandLine.
- * @throws IOException throws exception.
- */
- public abstract void execute(CommandLine cmd) throws IOException;
-
- /**
- * Display a help message describing the options the command takes.
- * TODO : currently only prints to standard out, may want to change this.
- */
- public abstract void displayHelp();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
deleted file mode 100644
index 7b8704a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/ResultCode.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-/**
- * The possible result code of SCM CLI.
- */
-public final class ResultCode {
- public static final int SUCCESS = 1;
-
- public static final int UNRECOGNIZED_CMD = 2;
-
- public static final int EXECUTION_ERROR = 3;
-
- private ResultCode() {}
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
deleted file mode 100644
index fb22d05..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SCMCLI.java
+++ /dev/null
@@ -1,231 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.ToolRunner;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.util.Arrays;
-
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.EXECUTION_ERROR;
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.SUCCESS;
-import static org.apache.hadoop.ozone.scm.cli.ResultCode.UNRECOGNIZED_CMD;
-import static org.apache.hadoop.ozone.scm.cli.container.ContainerCommandHandler.CONTAINER_CMD;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
-
-/**
- * This class is the CLI of SCM.
- */
-public class SCMCLI extends OzoneBaseCLI {
-
- public static final String HELP_OP = "help";
- public static final int CMD_WIDTH = 80;
-
- private final ScmClient scmClient;
- private final PrintStream out;
- private final PrintStream err;
-
- private final Options options;
-
- public SCMCLI(ScmClient scmClient) {
- this(scmClient, System.out, System.err);
- }
-
- public SCMCLI(ScmClient scmClient, PrintStream out, PrintStream err) {
- this.scmClient = scmClient;
- this.out = out;
- this.err = err;
- this.options = getOptions();
- }
-
- /**
- * Main for the scm shell Command handling.
- *
- * @param argv - System Args Strings[]
- * @throws Exception
- */
- public static void main(String[] argv) throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- ScmClient scmClient = getScmClient(conf);
- SCMCLI shell = new SCMCLI(scmClient);
- conf.setQuietMode(false);
- shell.setConf(conf);
- int res = 0;
- try {
- res = ToolRunner.run(shell, argv);
- } catch (Exception ex) {
- System.exit(1);
- }
- System.exit(res);
- }
-
- private static ScmClient getScmClient(OzoneConfiguration ozoneConf)
- throws IOException {
- long version = RPC.getProtocolVersion(
- StorageContainerLocationProtocolPB.class);
- InetSocketAddress scmAddress =
- OzoneClientUtils.getScmAddressForClients(ozoneConf);
- int containerSizeGB = ozoneConf.getInt(OZONE_SCM_CONTAINER_SIZE_GB,
- OZONE_SCM_CONTAINER_SIZE_DEFAULT);
- ContainerOperationClient.setContainerSizeB(containerSizeGB*OzoneConsts.GB);
-
- RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- StorageContainerLocationProtocolClientSideTranslatorPB client =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
- scmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
- NetUtils.getDefaultSocketFactory(ozoneConf),
- Client.getRpcTimeout(ozoneConf)));
- ScmClient storageClient = new ContainerOperationClient(
- client, new XceiverClientManager(ozoneConf));
- return storageClient;
- }
-
- /**
- * Adds ALL the options that hdfs scm command supports. Given the hierarchy
- * of commands, the options are added in a cascading manner, e.g.:
- * {@link SCMCLI} asks {@link ContainerCommandHandler} to add it's options,
- * which then asks it's sub command, such as
- * {@link org.apache.hadoop.ozone.scm.cli.container.CreateContainerHandler}
- * to add it's own options.
- *
- * We need to do this because {@link BasicParser} need to take all the options
- * when paring args.
- * @return ALL the options supported by this CLI.
- */
- @Override
- protected Options getOptions() {
- Options newOptions = new Options();
- // add the options
- addTopLevelOptions(newOptions);
- ContainerCommandHandler.addOptions(newOptions);
- // TODO : add pool, node and pipeline commands.
- addHelpOption(newOptions);
- return newOptions;
- }
-
- private static void addTopLevelOptions(Options options) {
- Option containerOps = new Option(
- CONTAINER_CMD, false, "Container related options");
- options.addOption(containerOps);
- // TODO : add pool, node and pipeline commands.
- }
-
- private static void addHelpOption(Options options) {
- Option helpOp = new Option(HELP_OP, false, "display help message");
- options.addOption(helpOp);
- }
-
- @Override
- protected void displayHelp() {
- HelpFormatter helpFormatter = new HelpFormatter();
- Options topLevelOptions = new Options();
- addTopLevelOptions(topLevelOptions);
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scmcli <commands> [<options>]",
- "where <commands> can be one of the following",
- topLevelOptions, "");
- }
-
- @Override
- public int run(String[] args) throws Exception {
- CommandLine cmd = parseArgs(args, options);
- if (cmd == null) {
- err.println("Unrecognized options:" + Arrays.asList(args));
- displayHelp();
- return UNRECOGNIZED_CMD;
- }
- return dispatch(cmd, options);
- }
-
- /**
- * This function parses all command line arguments
- * and returns the appropriate values.
- *
- * @param argv - Argv from main
- *
- * @return CommandLine
- */
- @Override
- protected CommandLine parseArgs(String[] argv, Options opts)
- throws ParseException {
- try {
- BasicParser parser = new BasicParser();
- return parser.parse(opts, argv);
- } catch (ParseException ex) {
- err.println(ex.getMessage());
- }
- return null;
- }
-
- @Override
- protected int dispatch(CommandLine cmd, Options opts)
- throws IOException, OzoneException, URISyntaxException {
- OzoneCommandHandler handler = null;
- try {
- if (cmd.hasOption(CONTAINER_CMD)) {
- handler = new ContainerCommandHandler(scmClient);
- }
-
- if (handler == null) {
- if (cmd.hasOption(HELP_OP)) {
- displayHelp();
- return SUCCESS;
- } else {
- displayHelp();
- err.println("Unrecognized command: " + Arrays.asList(cmd.getArgs()));
- return UNRECOGNIZED_CMD;
- }
- } else {
- // Redirect stdout and stderr if necessary.
- handler.setOut(this.out);
- handler.setErr(this.err);
- handler.execute(cmd);
- return SUCCESS;
- }
- } catch (IOException ioe) {
- err.println("Error executing command:" + ioe);
- return EXECUTION_ERROR;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
deleted file mode 100644
index 6152c55..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/SQLCLI.java
+++ /dev/null
@@ -1,720 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.DFSUtilClient;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.Pipeline;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_SUFFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_USER_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_BUCKET_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_VOLUME_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.NODEPOOL_DB;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_CONTAINERS_DB;
-
-/**
- * This is the CLI that can be use to convert an ozone metadata DB into
- * a sqlite DB file.
- *
- * NOTE: user should use this CLI in an offline fashion. Namely, this should not
- * be used to convert a DB that is currently being used by Ozone. Instead,
- * this should be used to debug and diagnosis closed DB instances.
- *
- */
-public class SQLCLI extends Configured implements Tool {
-
- private Options options;
- private BasicParser parser;
- private final Charset encoding = Charset.forName("UTF-8");
- private final OzoneConfiguration conf;
-
- // for container.db
- private static final String CREATE_CONTAINER_INFO =
- "CREATE TABLE containerInfo (" +
- "containerName TEXT PRIMARY KEY NOT NULL, " +
- "leaderUUID TEXT NOT NULL)";
- private static final String CREATE_CONTAINER_MEMBERS =
- "CREATE TABLE containerMembers (" +
- "containerName TEXT NOT NULL, " +
- "datanodeUUID TEXT NOT NULL," +
- "PRIMARY KEY(containerName, datanodeUUID));";
- private static final String CREATE_DATANODE_INFO =
- "CREATE TABLE datanodeInfo (" +
- "hostName TEXT NOT NULL, " +
- "datanodeUUId TEXT PRIMARY KEY NOT NULL," +
- "ipAddr TEXT, " +
- "xferPort INTEGER," +
- "infoPort INTEGER," +
- "ipcPort INTEGER," +
- "infoSecurePort INTEGER," +
- "containerPort INTEGER NOT NULL);";
- private static final String INSERT_CONTAINER_INFO =
- "INSERT INTO containerInfo (containerName, leaderUUID) " +
- "VALUES (\"%s\", \"%s\")";
- private static final String INSERT_DATANODE_INFO =
- "INSERT INTO datanodeInfo (hostname, datanodeUUid, ipAddr, xferPort, " +
- "infoPort, ipcPort, infoSecurePort, containerPort) " +
- "VALUES (\"%s\", \"%s\", \"%s\", %d, %d, %d, %d, %d)";
- private static final String INSERT_CONTAINER_MEMBERS =
- "INSERT INTO containerMembers (containerName, datanodeUUID) " +
- "VALUES (\"%s\", \"%s\")";
- // for block.db
- private static final String CREATE_BLOCK_CONTAINER =
- "CREATE TABLE blockContainer (" +
- "blockKey TEXT PRIMARY KEY NOT NULL, " +
- "containerName TEXT NOT NULL)";
- private static final String INSERT_BLOCK_CONTAINER =
- "INSERT INTO blockContainer (blockKey, containerName) " +
- "VALUES (\"%s\", \"%s\")";
- // for nodepool.db
- private static final String CREATE_NODE_POOL =
- "CREATE TABLE nodePool (" +
- "datanodeUUID TEXT NOT NULL," +
- "poolName TEXT NOT NULL," +
- "PRIMARY KEY(datanodeUUID, poolName))";
- private static final String INSERT_NODE_POOL =
- "INSERT INTO nodePool (datanodeUUID, poolName) " +
- "VALUES (\"%s\", \"%s\")";
- // and reuse CREATE_DATANODE_INFO and INSERT_DATANODE_INFO
- // for openContainer.db
- private static final String CREATE_OPEN_CONTAINER =
- "CREATE TABLE openContainer (" +
- "containerName TEXT PRIMARY KEY NOT NULL, " +
- "containerUsed INTEGER NOT NULL)";
- private static final String INSERT_OPEN_CONTAINER =
- "INSERT INTO openContainer (containerName, containerUsed) " +
- "VALUES (\"%s\", \"%s\")";
-
- // for ksm.db
- private static final String CREATE_VOLUME_LIST =
- "CREATE TABLE volumeList (" +
- "userName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "PRIMARY KEY (userName, volumeName))";
- private static final String INSERT_VOLUME_LIST =
- "INSERT INTO volumeList (userName, volumeName) " +
- "VALUES (\"%s\", \"%s\")";
-
- private static final String CREATE_VOLUME_INFO =
- "CREATE TABLE volumeInfo (" +
- "adminName TEXT NOT NULL," +
- "ownerName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "PRIMARY KEY (adminName, ownerName, volumeName))";
- private static final String INSERT_VOLUME_INFO =
- "INSERT INTO volumeInfo (adminName, ownerName, volumeName) " +
- "VALUES (\"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_ACL_INFO =
- "CREATE TABLE aclInfo (" +
- "adminName TEXT NOT NULL," +
- "ownerName TEXT NOT NULL," +
- "volumeName TEXT NOT NULL," +
- "type TEXT NOT NULL," +
- "userName TEXT NOT NULL," +
- "rights TEXT NOT NULL," +
- "FOREIGN KEY (adminName, ownerName, volumeName, userName, type)" +
- "REFERENCES " +
- "volumeInfo(adminName, ownerName, volumeName, userName, type)" +
- "PRIMARY KEY (adminName, ownerName, volumeName, userName, type))";
- private static final String INSERT_ACL_INFO =
- "INSERT INTO aclInfo (adminName, ownerName, volumeName, type, " +
- "userName, rights) " +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_BUCKET_INFO =
- "CREATE TABLE bucketInfo (" +
- "volumeName TEXT NOT NULL," +
- "bucketName TEXT NOT NULL," +
- "versionEnabled BOOLEAN NOT NULL," +
- "storageType TEXT," +
- "PRIMARY KEY (volumeName, bucketName))";
- private static final String INSERT_BUCKET_INFO =
- "INSERT INTO bucketInfo(volumeName, bucketName, " +
- "versionEnabled, storageType)" +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final String CREATE_KEY_INFO =
- "CREATE TABLE keyInfo (" +
- "volumeName TEXT NOT NULL," +
- "bucketName TEXT NOT NULL," +
- "keyName TEXT NOT NULL," +
- "dataSize INTEGER," +
- "blockKey TEXT NOT NULL," +
- "containerName TEXT NOT NULL," +
- "PRIMARY KEY (volumeName, bucketName, keyName))";
- private static final String INSERT_KEY_INFO =
- "INSERT INTO keyInfo (volumeName, bucketName, keyName, dataSize, " +
- "blockKey, containerName)" +
- "VALUES (\"%s\", \"%s\", \"%s\", \"%s\", \"%s\", \"%s\")";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(SQLCLI.class);
-
- public SQLCLI(OzoneConfiguration conf) {
- this.options = getOptions();
- this.parser = new BasicParser();
- this.conf = conf;
- }
-
- @SuppressWarnings("static-access")
- private Options getOptions() {
- Options allOptions = new Options();
- Option helpOpt = OptionBuilder
- .hasArg(false)
- .withLongOpt("help")
- .withDescription("display help message")
- .create("h");
- allOptions.addOption(helpOpt);
-
- Option dbPathOption = OptionBuilder
- .withArgName("DB path")
- .withLongOpt("dbPath")
- .hasArgs(1)
- .withDescription("specify DB path")
- .create("p");
- allOptions.addOption(dbPathOption);
-
- Option outPathOption = OptionBuilder
- .withArgName("output path")
- .withLongOpt("outPath")
- .hasArgs(1)
- .withDescription("specify output DB file path")
- .create("o");
- allOptions.addOption(outPathOption);
-
- return allOptions;
- }
-
- public void displayHelp() {
- HelpFormatter helpFormatter = new HelpFormatter();
- Options allOpts = getOptions();
- helpFormatter.printHelp("hdfs oz_debug -p <DB path>"
- + " -o <Output DB file path>", allOpts);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- CommandLine commandLine = parseArgs(args);
- if (commandLine.hasOption("help")) {
- displayHelp();
- return 0;
- }
- if (!commandLine.hasOption("p") || !commandLine.hasOption("o")) {
- displayHelp();
- return -1;
- }
- String value = commandLine.getOptionValue("p");
- LOG.info("DB path {}", value);
- // the value is supposed to be an absolute path to a container file
- Path dbPath = Paths.get(value);
- if (!Files.exists(dbPath)) {
- LOG.error("DB path not exist:{}", dbPath);
- }
- Path parentPath = dbPath.getParent();
- Path dbName = dbPath.getFileName();
- if (parentPath == null || dbName == null) {
- LOG.error("Error processing db path {}", dbPath);
- return -1;
- }
-
- value = commandLine.getOptionValue("o");
- Path outPath = Paths.get(value);
- if (outPath == null || outPath.getParent() == null) {
- LOG.error("Error processing output path {}", outPath);
- return -1;
- }
-
- if (outPath.toFile().isDirectory()) {
- LOG.error("The db output path should be a file instead of a directory");
- return -1;
- }
-
- Path outParentPath = outPath.getParent();
- if (outParentPath != null) {
- if (!Files.exists(outParentPath)) {
- Files.createDirectories(outParentPath);
- }
- }
- LOG.info("Parent path [{}] db name [{}]", parentPath, dbName);
- if (dbName.toString().endsWith(CONTAINER_DB_SUFFIX)) {
- LOG.info("Converting container DB");
- convertContainerDB(dbPath, outPath);
- } else if (dbName.toString().equals(BLOCK_DB)) {
- LOG.info("Converting block DB");
- convertBlockDB(dbPath, outPath);
- } else if (dbName.toString().equals(NODEPOOL_DB)) {
- LOG.info("Converting node pool DB");
- convertNodePoolDB(dbPath, outPath);
- } else if (dbName.toString().equals(OPEN_CONTAINERS_DB)) {
- LOG.info("Converting open container DB");
- convertOpenContainerDB(dbPath, outPath);
- } else if (dbName.toString().equals(KSM_DB_NAME)) {
- LOG.info("Converting ksm DB");
- convertKSMDB(dbPath, outPath);
- } else {
- LOG.error("Unrecognized db name {}", dbName);
- }
- return 0;
- }
-
- private Connection connectDB(String dbPath) throws Exception {
- Class.forName("org.sqlite.JDBC");
- String connectPath =
- String.format("jdbc:sqlite:%s", dbPath);
- return DriverManager.getConnection(connectPath);
- }
-
- private void executeSQL(Connection conn, String sql) throws SQLException {
- try (Statement stmt = conn.createStatement()) {
- stmt.executeUpdate(sql);
- }
- }
-
- /**
- * Convert ksm.db to sqlite db file. With following schema.
- * (* for primary key)
- *
- * 1. for key type USER, it contains a username and a list volumes
- * volumeList
- * --------------------------------
- * userName* | volumeName*
- * --------------------------------
- *
- * 2. for key type VOLUME:
- *
- * volumeInfo
- * ----------------------------------------------
- * adminName | ownerName* | volumeName* | aclID
- * ----------------------------------------------
- *
- * aclInfo
- * ----------------------------------------------
- * aclEntryID* | type* | userName* | rights
- * ----------------------------------------------
- *
- * 3. for key type BUCKET
- * bucketInfo
- * --------------------------------------------------------
- * volumeName* | bucketName* | versionEnabled | storageType
- * --------------------------------------------------------
- *
- * TODO : the following table will be changed when key partition is added.
- * Only has the minimum entries for test purpose now.
- * 4. for key type KEY
- * -----------------------------------------------
- * volumeName* | bucketName* | keyName* | dataSize
- * -----------------------------------------------
- *
- *
- *
- * @param dbPath
- * @param outPath
- * @throws Exception
- */
- private void convertKSMDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create tables for sql ksm db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_VOLUME_LIST);
- executeSQL(conn, CREATE_VOLUME_INFO);
- executeSQL(conn, CREATE_ACL_INFO);
- executeSQL(conn, CREATE_BUCKET_INFO);
- executeSQL(conn, CREATE_KEY_INFO);
-
- dbStore.iterate(null, (key, value) -> {
- String keyString = DFSUtilClient.bytes2String(key);
- KeyType type = getKeyType(keyString);
- try {
- insertKSMDB(conn, type, keyString, value);
- } catch (IOException | SQLException ex) {
- LOG.error("Exception inserting key {} type {}", keyString, type, ex);
- }
- return true;
- });
- }
- }
-
- private void insertKSMDB(Connection conn, KeyType type, String keyName,
- byte[] value) throws IOException, SQLException {
- switch (type) {
- case USER:
- VolumeList volumeList = VolumeList.parseFrom(value);
- for (String volumeName : volumeList.getVolumeNamesList()) {
- String insertVolumeList =
- String.format(INSERT_VOLUME_LIST, keyName, volumeName);
- executeSQL(conn, insertVolumeList);
- }
- break;
- case VOLUME:
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(value);
- String adminName = volumeInfo.getAdminName();
- String ownerName = volumeInfo.getOwnerName();
- String volumeName = volumeInfo.getVolume();
- String insertVolumeInfo =
- String.format(INSERT_VOLUME_INFO, adminName, ownerName, volumeName);
- executeSQL(conn, insertVolumeInfo);
- for (OzoneAclInfo aclInfo : volumeInfo.getVolumeAclsList()) {
- String insertAclInfo =
- String.format(INSERT_ACL_INFO, adminName, ownerName, volumeName,
- aclInfo.getType(), aclInfo.getName(), aclInfo.getRights());
- executeSQL(conn, insertAclInfo);
- }
- break;
- case BUCKET:
- BucketInfo bucketInfo = BucketInfo.parseFrom(value);
- String insertBucketInfo =
- String.format(INSERT_BUCKET_INFO, bucketInfo.getVolumeName(),
- bucketInfo.getBucketName(), bucketInfo.getIsVersionEnabled(),
- bucketInfo.getStorageType());
- executeSQL(conn, insertBucketInfo);
- break;
- case KEY:
- KeyInfo keyInfo = KeyInfo.parseFrom(value);
- // TODO : the two fields container name and block id are no longer used,
- // need to revisit this later.
- String insertKeyInfo =
- String.format(INSERT_KEY_INFO, keyInfo.getVolumeName(),
- keyInfo.getBucketName(), keyInfo.getKeyName(),
- keyInfo.getDataSize(), "EMPTY",
- "EMPTY");
- executeSQL(conn, insertKeyInfo);
- break;
- default:
- throw new IOException("Unknown key from ksm.db");
- }
- }
-
- private KeyType getKeyType(String key) {
- if (key.startsWith(KSM_USER_PREFIX)) {
- return KeyType.USER;
- } else if (key.startsWith(KSM_VOLUME_PREFIX)) {
- return key.replaceFirst(KSM_VOLUME_PREFIX, "")
- .contains(KSM_BUCKET_PREFIX) ? KeyType.BUCKET : KeyType.VOLUME;
- }else {
- return KeyType.KEY;
- }
- }
-
- private enum KeyType {
- USER,
- VOLUME,
- BUCKET,
- KEY,
- UNKNOWN
- }
-
- /**
- * Convert container.db to sqlite. The schema of sql db:
- * three tables, containerId, containerMachines, datanodeInfo
- * (* for primary key)
- *
- * containerInfo:
- * ----------------------------------------------
- * container name* | container lead datanode uuid
- * ----------------------------------------------
- *
- * containerMembers:
- * --------------------------------
- * container name* | datanodeUUid*
- * --------------------------------
- *
- * datanodeInfo:
- * ---------------------------------------------------------
- * hostname | datanodeUUid* | xferPort | infoPort | ipcPort
- * ---------------------------------------------------------
- *
- * --------------------------------
- * | infoSecurePort | containerPort
- * --------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertContainerDB(Path dbPath, Path outPath)
- throws Exception {
- LOG.info("Create tables for sql container db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_CONTAINER_INFO);
- executeSQL(conn, CREATE_CONTAINER_MEMBERS);
- executeSQL(conn, CREATE_DATANODE_INFO);
-
- HashSet<String> uuidChecked = new HashSet<>();
- dbStore.iterate(null, (key, value) -> {
- String containerName = new String(key, encoding);
- ContainerInfo containerInfo = null;
- containerInfo = ContainerInfo.fromProtobuf(
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(value));
- Preconditions.checkNotNull(containerInfo);
- try {
- //TODO: include container state to sqllite schema
- insertContainerDB(conn, containerName,
- containerInfo.getPipeline().getProtobufMessage(), uuidChecked);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- /**
- * Insert into the sqlite DB of container.db.
- * @param conn the connection to the sqlite DB.
- * @param containerName the name of the container.
- * @param pipeline the actual container pipeline object.
- * @param uuidChecked the uuid that has been already inserted.
- * @throws SQLException throws exception.
- */
- private void insertContainerDB(Connection conn, String containerName,
- Pipeline pipeline, Set<String> uuidChecked) throws SQLException {
- LOG.info("Insert to sql container db, for container {}", containerName);
- String insertContainerInfo = String.format(
- INSERT_CONTAINER_INFO, containerName,
- pipeline.getPipelineChannel().getLeaderID());
- executeSQL(conn, insertContainerInfo);
-
- for (HdfsProtos.DatanodeIDProto dnID :
- pipeline.getPipelineChannel().getMembersList()) {
- String uuid = dnID.getDatanodeUuid();
- if (!uuidChecked.contains(uuid)) {
- // we may also not use this checked set, but catch exception instead
- // but this seems a bit cleaner.
- String ipAddr = dnID.getIpAddr();
- String hostName = dnID.getHostName();
- int xferPort = dnID.hasXferPort() ? dnID.getXferPort() : 0;
- int infoPort = dnID.hasInfoPort() ? dnID.getInfoPort() : 0;
- int securePort =
- dnID.hasInfoSecurePort() ? dnID.getInfoSecurePort() : 0;
- int ipcPort = dnID.hasIpcPort() ? dnID.getIpcPort() : 0;
- int containerPort = dnID.getContainerPort();
- String insertMachineInfo = String.format(
- INSERT_DATANODE_INFO, hostName, uuid, ipAddr, xferPort, infoPort,
- ipcPort, securePort, containerPort);
- executeSQL(conn, insertMachineInfo);
- uuidChecked.add(uuid);
- }
- String insertContainerMembers = String.format(
- INSERT_CONTAINER_MEMBERS, containerName, uuid);
- executeSQL(conn, insertContainerMembers);
- }
- LOG.info("Insertion completed.");
- }
-
- /**
- * Converts block.db to sqlite. This is rather simple db, the schema has only
- * one table:
- *
- * blockContainer
- * --------------------------
- * blockKey* | containerName
- * --------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertBlockDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create tables for sql block db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_BLOCK_CONTAINER);
-
- dbStore.iterate(null, (key, value) -> {
- String blockKey = DFSUtilClient.bytes2String(key);
- String containerName = DFSUtilClient.bytes2String(value);
- String insertBlockContainer = String.format(
- INSERT_BLOCK_CONTAINER, blockKey, containerName);
-
- try {
- executeSQL(conn, insertBlockContainer);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- /**
- * Converts nodePool.db to sqlite. The schema of sql db:
- * two tables, nodePool and datanodeInfo (the same datanode Info as for
- * container.db).
- *
- * nodePool
- * ---------------------------------------------------------
- * datanodeUUID* | poolName*
- * ---------------------------------------------------------
- *
- * datanodeInfo:
- * ---------------------------------------------------------
- * hostname | datanodeUUid* | xferPort | infoPort | ipcPort
- * ---------------------------------------------------------
- *
- * --------------------------------
- * | infoSecurePort | containerPort
- * --------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertNodePoolDB(Path dbPath, Path outPath) throws Exception {
- LOG.info("Create table for sql node pool db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_NODE_POOL);
- executeSQL(conn, CREATE_DATANODE_INFO);
-
- dbStore.iterate(null, (key, value) -> {
- DatanodeID nodeId = DatanodeID
- .getFromProtoBuf(HdfsProtos.DatanodeIDProto.PARSER.parseFrom(key));
- String blockPool = DFSUtil.bytes2String(value);
- try {
- insertNodePoolDB(conn, blockPool, nodeId);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- private void insertNodePoolDB(Connection conn, String blockPool,
- DatanodeID datanodeID) throws SQLException {
- String insertNodePool = String.format(INSERT_NODE_POOL,
- datanodeID.getDatanodeUuid(), blockPool);
- executeSQL(conn, insertNodePool);
-
- String insertDatanodeID = String.format(INSERT_DATANODE_INFO,
- datanodeID.getHostName(), datanodeID.getDatanodeUuid(),
- datanodeID.getIpAddr(), datanodeID.getXferPort(),
- datanodeID.getInfoPort(), datanodeID.getIpcPort(),
- datanodeID.getInfoSecurePort(), datanodeID.getContainerPort());
- executeSQL(conn, insertDatanodeID);
- }
-
- /**
- * Convert openContainer.db to sqlite db file. This is rather simple db,
- * the schema has only one table:
- *
- * openContainer
- * -------------------------------
- * containerName* | containerUsed
- * -------------------------------
- *
- * @param dbPath path to container db.
- * @param outPath path to output sqlite
- * @throws IOException throws exception.
- */
- private void convertOpenContainerDB(Path dbPath, Path outPath)
- throws Exception {
- LOG.info("Create table for open container db.");
- File dbFile = dbPath.toFile();
- try (MetadataStore dbStore = MetadataStoreBuilder.newBuilder()
- .setConf(conf).setDbFile(dbFile).build();
- Connection conn = connectDB(outPath.toString())) {
- executeSQL(conn, CREATE_OPEN_CONTAINER);
-
- dbStore.iterate(null, (key, value) -> {
- String containerName = DFSUtil.bytes2String(key);
- Long containerUsed =
- Long.parseLong(DFSUtil.bytes2String(value));
- String insertOpenContainer = String
- .format(INSERT_OPEN_CONTAINER, containerName, containerUsed);
- try {
- executeSQL(conn, insertOpenContainer);
- return true;
- } catch (SQLException e) {
- throw new IOException(e);
- }
- });
- }
- }
-
- private CommandLine parseArgs(String[] argv)
- throws ParseException {
- return parser.parse(options, argv);
- }
-
- public static void main(String[] args) {
- Tool shell = new SQLCLI(new OzoneConfiguration());
- int res = 0;
- try {
- ToolRunner.run(shell, args);
- } catch (Exception ex) {
- LOG.error(ex.toString());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Command execution failed", ex);
- }
- res = 1;
- }
- System.exit(res);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
deleted file mode 100644
index 7a52305..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CloseContainerHandler.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * The handler of close container command.
- */
-public class CloseContainerHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_CLOSE = "close";
- public static final String OPT_CONTAINER_NAME = "c";
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- if (!cmd.hasOption(CONTAINER_CLOSE)) {
- throw new IOException("Expecting container close");
- }
- if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container name");
- } else {
- return;
- }
- }
- String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
- Pipeline pipeline = getScmClient().getContainer(containerName);
- if (pipeline == null) {
- throw new IOException("Cannot close an non-exist container "
- + containerName);
- }
- logOut("Closing container : %s.", containerName);
- getScmClient().closeContainer(pipeline);
- logOut("Container closed.");
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -close <option>",
- "where <option> is", options, "");
- }
-
- public static void addOptions(Options options) {
- Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
- true, "Specify container name");
- options.addOption(containerNameOpt);
- }
-
- CloseContainerHandler(ScmClient client) {
- super(client);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
deleted file mode 100644
index 7838b1f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ContainerCommandHandler.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-import java.util.Arrays;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-import static org.apache.hadoop.ozone.scm.cli.container.CloseContainerHandler.CONTAINER_CLOSE;
-import static org.apache.hadoop.ozone.scm.cli.container
- .CreateContainerHandler.CONTAINER_CREATE;
-import static org.apache.hadoop.ozone.scm.cli.container
- .DeleteContainerHandler.CONTAINER_DELETE;
-import static org.apache.hadoop.ozone.scm.cli.container
- .InfoContainerHandler.CONTAINER_INFO;
-import static org.apache.hadoop.ozone.scm.cli.container
- .ListContainerHandler.CONTAINER_LIST;
-
-/**
- * The handler class of container-specific commands, e.g. addContainer.
- */
-public class ContainerCommandHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_CMD = "container";
-
- public ContainerCommandHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- // all container commands should contain -container option
- if (!cmd.hasOption(CONTAINER_CMD)) {
- throw new IOException("Expecting container cmd");
- }
- // check which each the sub command it is
- OzoneCommandHandler handler = null;
- if (cmd.hasOption(CONTAINER_CREATE)) {
- handler = new CreateContainerHandler(getScmClient());
- } else if (cmd.hasOption(CONTAINER_DELETE)) {
- handler = new DeleteContainerHandler(getScmClient());
- } else if (cmd.hasOption(CONTAINER_INFO)) {
- handler = new InfoContainerHandler(getScmClient());
- } else if (cmd.hasOption(CONTAINER_LIST)) {
- handler = new ListContainerHandler(getScmClient());
- } else if (cmd.hasOption(CONTAINER_CLOSE)) {
- handler = new CloseContainerHandler(getScmClient());
- }
-
- // execute the sub command, throw exception if no sub command found
- // unless -help option is given.
- if (handler != null) {
- handler.setOut(this.out);
- handler.setErr(this.err);
- handler.execute(cmd);
- } else {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Unrecognized command "
- + Arrays.asList(cmd.getArgs()));
- }
- }
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addCommandsOption(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH,
- "hdfs scm -container <commands> <options>",
- "where <commands> can be one of the following", options, "");
- }
-
- private static void addCommandsOption(Options options) {
- Option createContainer =
- new Option(CONTAINER_CREATE, false, "Create container");
- Option infoContainer =
- new Option(CONTAINER_INFO, false, "Info container");
- Option deleteContainer =
- new Option(CONTAINER_DELETE, false, "Delete container");
- Option listContainer =
- new Option(CONTAINER_LIST, false, "List container");
- Option closeContainer =
- new Option(CONTAINER_CLOSE, false, "Close container");
-
- options.addOption(createContainer);
- options.addOption(deleteContainer);
- options.addOption(infoContainer);
- options.addOption(listContainer);
- options.addOption(closeContainer);
- // Every new option should add it's option here.
- }
-
- public static void addOptions(Options options) {
- addCommandsOption(options);
- // for create container options.
- CreateContainerHandler.addOptions(options);
- DeleteContainerHandler.addOptions(options);
- InfoContainerHandler.addOptions(options);
- ListContainerHandler.addOptions(options);
- CloseContainerHandler.addOptions(options);
- // Every new option should add it's option here.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
deleted file mode 100644
index 569c4a3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/CreateContainerHandler.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container creation command.
- */
-public class CreateContainerHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_CREATE = "create";
- public static final String OPT_CONTAINER_NAME = "c";
- public static final String containerOwner = "OZONE";
- // TODO Support an optional -p <pipelineID> option to create
- // container on given datanodes.
-
- public CreateContainerHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- if (!cmd.hasOption(CONTAINER_CREATE)) {
- throw new IOException("Expecting container create");
- }
- if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container name");
- } else {
- return;
- }
- }
- String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
- logOut("Creating container : %s.", containerName);
- getScmClient().createContainer(containerName, containerOwner);
- logOut("Container created.");
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -create <option>",
- "where <option> is", options, "");
- }
-
- public static void addOptions(Options options) {
- Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
- true, "Specify container name");
- options.addOption(containerNameOpt);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
deleted file mode 100644
index 0782af3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/DeleteContainerHandler.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process delete container command.
- */
-public class DeleteContainerHandler extends OzoneCommandHandler {
-
- protected static final String CONTAINER_DELETE = "delete";
- protected static final String OPT_FORCE = "f";
- protected static final String OPT_CONTAINER_NAME = "c";
-
- public DeleteContainerHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- Preconditions.checkArgument(cmd.hasOption(CONTAINER_DELETE),
- "Expecting command delete");
- if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container name");
- } else {
- return;
- }
- }
-
- String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
-
- Pipeline pipeline = getScmClient().getContainer(containerName);
- if (pipeline == null) {
- throw new IOException("Cannot delete an non-exist container "
- + containerName);
- }
-
- logOut("Deleting container : %s.", containerName);
- getScmClient().deleteContainer(pipeline, cmd.hasOption(OPT_FORCE));
- logOut("Container %s deleted.", containerName);
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -delete <option>",
- "where <option> is", options, "");
- }
-
- public static void addOptions(Options options) {
- Option forceOpt = new Option(OPT_FORCE,
- false,
- "forcibly delete a container");
- options.addOption(forceOpt);
- Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
- true, "Specify container name");
- options.addOption(containerNameOpt);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
deleted file mode 100644
index f54a2b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/InfoContainerHandler.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container info command.
- */
-public class InfoContainerHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_INFO = "info";
- protected static final String OPT_CONTAINER_NAME = "c";
-
- /**
- * Constructs a handler object.
- *
- * @param scmClient scm client.
- */
- public InfoContainerHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- if (!cmd.hasOption(CONTAINER_INFO)) {
- throw new IOException("Expecting container info");
- }
- if (!cmd.hasOption(OPT_CONTAINER_NAME)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container name");
- } else {
- return;
- }
- }
- String containerName = cmd.getOptionValue(OPT_CONTAINER_NAME);
- Pipeline pipeline = getScmClient().getContainer(containerName);
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
-
- ContainerData containerData =
- getScmClient().readContainer(pipeline);
-
- // Print container report info.
- logOut("Container Name: %s",
- containerData.getName());
- String openStatus =
- containerData.getState() == OzoneProtos.LifeCycleState.OPEN ? "OPEN" :
- "CLOSED";
- logOut("Container State: %s", openStatus);
- if (!containerData.getHash().isEmpty()) {
- logOut("Container Hash: %s", containerData.getHash());
- }
- logOut("Container DB Path: %s", containerData.getDbPath());
- logOut("Container Path: %s", containerData.getContainerPath());
-
- // Output meta data.
- String metadataStr = containerData.getMetadataList().stream().map(
- p -> p.getKey() + ":" + p.getValue()).collect(Collectors.joining(", "));
- logOut("Container Metadata: {%s}", metadataStr);
-
- // Print pipeline of an existing container.
- logOut("LeaderID: %s", pipeline.getLeader().getHostName());
- String machinesStr = pipeline.getMachines().stream().map(
- DatanodeID::getHostName).collect(Collectors.joining(","));
- logOut("Datanodes: [%s]", machinesStr);
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -info <option>",
- "where <option> is", options, "");
- }
-
- public static void addOptions(Options options) {
- Option containerNameOpt = new Option(OPT_CONTAINER_NAME,
- true, "Specify container name");
- options.addOption(containerNameOpt);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
deleted file mode 100644
index 594da5a3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"file":"nv.d3.min.js","sources":["../src/core.js","../src/dom.js","../src/interactiveLayer.js","../src/tooltip.js","../src/utils.js","../src/models/axis.js","../src/models/boxPlot.js","../src/models/boxPlotChart.js","../src/models/bullet.js","../src/models/bulletChart.js","../src/models/candlestickBar.js","../src/models/cumulativeLineChart.js","../src/models/discreteBar.js","../src/models/discreteBarChart.js","../src/models/distribution.js","../src/models/focus.js","../src/models/forceDirectedGraph.js","../src/models/furiousLegend.js","../src/models/historicalBar.js","../src/models/historicalBarChart.js","../src/models/legend.js","../src/models/line.js","../src/models/lineChart.js","../src/models/linePlusBarChart.js","../src/models/multiBar.js","../src/models/multiBarChart.js","../src/models/multiBarHorizontal.js","../src/models/multiBarHorizontalChart.js","../src/models/multiChart.js","../src/models/ohlcBar.js","../src/models/parallelCoordinates.js","../src/models/para
llelCoordinatesChart.js","../src/models/pie.js","../src/models/pieChart.js","../src/models/sankey.js","../src/models/sankeyChart.js","../src/models/scatter.js","../src/models/scatterChart.js","../src/models/sparkline.js","../src/models/sparklinePlus.js","../src/models/stackedArea.js","../src/models/stackedAreaChart.js","../src/models/sunburst.js","../src/models/sunburstChart.js"],"names":["nv","dev","tooltip","utils","models","charts","logs","dom","d3","require","dispatch","Function","prototype","bind","oThis","this","TypeError","aArgs","Array","slice","call","arguments","fToBind","fNOP","fBound","apply","concat","on","e","startTime","Date","endTime","totalTime","log","window","console","length","deprecated","name","info","warn","render","step","active","render_start","renderLoop","chart","graph","i","queue","generate","callback","splice","setTimeout","render_end","addGraph","obj","push","module","exports","write","undefined","fastdom","mutate","read","measure","interactiveGuideline
","layer","selection","each","data","mouseHandler","d3mouse","mouse","mouseX","mouseY","subtractMargin","mouseOutAnyReason","isMSIE","event","offsetX","offsetY","target","tagName","className","baseVal","match","margin","left","top","type","availableWidth","availableHeight","relatedTarget","ownerSVGElement","nvPointerEventsClass","elementMouseout","renderGuideLine","hidden","scaleIsOrdinal","xScale","rangeBands","pointXValue","elementIndex","bisect","range","rangeBand","domain","invert","elementMousemove","elementDblclick","elementClick","elementMouseDown","elementMouseUp","container","select","width","height","wrap","selectAll","wrapEnter","enter","append","attr","svgContainer","guideLine","x","showGuideLine","line","NaNtoZero","String","d","exit","remove","scale","linear","ActiveXObject","duration","hideDelay","_","interactiveBisect","values","searchVal","xAccessor","_xAccessor","_cmp","v","bisector","index","max","currentValue","nextIndex","min","nextValue","Math","abs","nearestVa
lueIndex","threshold","yDistMax","Infinity","indexToHighlight","forEach","delta","initTooltip","node","document","body","id","classes","style","classed","nvtooltip","enabled","dataSeriesExists","newContent","contentGenerator","innerHTML","positionTooltip","floor","random","gravity","distance","snapDistance","lastPosition","headerEnabled","valueFormatter","headerFormatter","keyFormatter","table","createElement","theadEnter","html","value","tbodyEnter","trowEnter","p","series","highlight","color","total","key","filter","percent","format","opacityScale","opacity","outerHTML","footer","position","pos","clientX","clientY","getComputedStyle","transform","client","getBoundingClientRect","isArray","isObject","calcGravityOffset","tmp","offsetHeight","offsetWidth","clientWidth","documentElement","clientHeight","gravityOffset","interrupt","transition","delay","old_translate","new_translate","round","translateInterpolator","interpolateString","is_hidden","styleTween","options","optionsFunc","_o
ptions","Object","create","get","set","chartContainer","fixedTop","offset","point","y","initOptions","windowSize","size","innerWidth","innerHeight","compatMode","a","isFunction","isDate","toString","isNumber","isNaN","windowResize","handler","addEventListener","clear","removeEventListener","getColor","defaultColor","color_scale","ordinal","category20","customTheme","dictionary","getKey","defaultColors","defIndex","pjax","links","content","load","href","fragment","parentNode","replaceChild","history","pushState","textContent","preventDefault","state","calcApproxTextWidth","svgTextElem","text","fontSize","parseInt","replace","textLength","n","watchTransition","renderWatch","args","_duration","renderStack","self","model","__rendered","m","arg","renderEnd","indexOf","reset","pop","every","deepExtend","dst","sources","source","srcObj","_setState","_getState","init","changed","getter","fn","setter","update","_set","settings","JSON","stringify","change","map","calcTicksX","numTicks","numVa
lues","stream_len","calcTicksY","initOption","_calls","_overrides","ops","getOwnPropertyNames","calls","inheritOptionsD3","d3_source","oplist","_d3options","unshift","rebind","arrayUnique","sort","item","symbolMap","symbol","t","s","svg","symbolTypes","functor","inheritOptions","inherited","_inherited","d3ops","initSVG","nvd3-svg","sanitizeHeight","sanitizeWidth","bottom","right","noData","opt","noDataText","wrapTicks","word","words","split","reverse","lineNumber","lineHeight","dy","parseFloat","tspan","join","getComputedTextLength","arrayEquals","array1","array2","l","axis","g","ticks","orient","scale0","fmt","tickFormat","axisLabel","axisLabelText","xLabelMargin","axisMaxMin","w","isOrdinal","showMaxMin","tickPadding","axisLabelDistance","maxTextWidth","textHeight","xTicks","rotateLabelsRule","rotateLabels","box","sin","PI","staggerLabels","rotateYLabel","maxMinRange","err","copy","boxPlot","xDomain","getX","xRange","yData","yDomain","yMin","yMax","q1","getQ1","q3","getQ3","wl","g
etWl","wh","getWh","olItems","getOlItems","getOlValue","yScale","yRange","xScale0","yScale0","boxplots","boxEnter","j","hover","f","box_width","maxBoxWidth","box_left","box_right","endpoint","elementMouseover","getQ2","outliers","getOlColor","getOlLabel","label","Q1","Q2","Q3","whisker_low","whisker_high","q2","itemColor","outlierValue","outlierLabel","outlierColor","boxPlotChart","boxplot","showXAxis","xAxis","showYAxis","yAxis","beforeUpdate","clamp","gEnter","defsEnter","rightAlignYAxis","barsWrap","datum","disabled","tickSize","evt","tooltipContent","bullet","sortLabels","labels","lz","b","iA","iB","descending","rangez","ranges","markerz","markers","markerLinez","markerLines","measurez","measures","rangeLabelz","rangeLabels","markerLabelz","markerLabels","markerLineLabelz","markerLineLabels","measureLabelz","measureLabels","x1","extent","merge","forceX","__chart__","il","rangeClassNames","legacyRangeClassNames","w1","xp1","h3","markerData","marker","markerLinesData","defaultRang
eLabels","bulletChart","x0","title","subtitle","bulletWrap","tick","tickEnter","tickUpdate","timer","flush","candlestickBar","barWidth","padData","getLow","forceY","getHigh","chartClick","clipEdge","tickGroups","getOpen","getClose","getY","open","close","high","low","interactive","highlightPoint","pointIndex","isHoverOver","clearHighlights","cumulativeLineChart","lines","dragStart","dragMove","dx","updateZero","dragEnd","stateChange","indexLine","oldDuration","stateSetter","stateGetter","defaultState","indexDrag","behavior","drag","rescaleY","seriesDomains","initialDomain","completeDomain","indexify","interactivePointerEvents","showLegend","legend","marginTop","showControls","controlsData","controls","rightAlign","tempDisabled","useInteractiveGuideline","interactiveLayer","display","linesWrap","seriesIndex","avgLineData","average","avgLines","getAvgLineY","yVal","_ticks","newState","singlePoint","pointXLocation","allData","yValue","domainExtent","xValue","idx","indexifyYGetter","ind
exValue","noErrorCheck","updateState","useVoronoi","discreteBar","seriesData","y0","showValues","groups","bars","barsEnter","element","stopPropagation","elementDblClick","valueFormat","rectClass","discreteBarChart","discretebar","wrapLabels","distribution","naxis","distWrap","dist","getData","focus","resizePath","updateBrushBG","brush","empty","brushExtent","brushBG","leftWidth","rightWidth","onBrush","shouldDispatch","contentWrap","syncBrushing","brushBGenter","gBrush","pointActive","interpolate","xTickFormat","yTickFormat","forceDirectedGraph","nodes","nodeFieldSet","Set","keys","add","force","layout","linkStrength","friction","linkDistance","linkDist","charge","theta","alpha","start","link","sqrt","radius","py","px","nodeColor","field","linkExtras","nodeExtras","furiousLegend","setTextColor","vers","expanded","disengaged","setBGColor","seriesShape","seriesEnter","property","seriesCheckbox","seriesText","legendMouseover","legendMouseout","legendClick","radioButtonMode","userDisabl
ed","engaged","legendDblclick","versPadding","align","seriesWidths","legendText","maxKeyLength","trimmedKey","substring","nodeTextLength","Error","padding","seriesPerRow","legendWidth","columnWidths","k","reduce","prev","cur","array","xPositions","curX","ceil","xpos","ypos","newxpos","maxwidth","historicalBar","rval","historicalBarChart","bar_model","transitionDuration","tooltipHide","ohlcBarChart","ohlcBar","candlestickBarChart","setBGOpacity","insert","seriesBG","scatter","scatterWrap","strokeWidth","fillOpacity","areaPaths","isArea","area","defined","y1","linePaths","pointSize","pointDomain","lineChart","updateXAxis","updateYAxis","focusLinesWrap","disableTooltip","focusEnable","focusEnter","legendPosition","currentValues","pointYValue","defaultValueFormatter","yPos","x2Axis","y2Axis","focusHeight","focusShowAxisX","focusShowAxisY","focusMargin","lineWithFocusChart","linePlusBarChart","availableHeight2","x2","availableHeight1","bar","focusBarsWrap","dataBars","allDisabled","dataL
ines","dataLine","switchYAxisOrder","y1Axis","y2","barsOpacity","linesOpacity","y1Opacity","y2Opacity","margin2","y3","lines2","bars2","y4","series1","series2","contextEnter","legendXPosition","originalKey","legendRightAxisHint","legendLeftAxisHint","bars2Wrap","lines2Wrap","y3Axis","y4Axis","getBarsAxis","main","getLinesAxis","multiBar","nonStackableCount","hideable","stacked","parsed","stack","stackOffset","nonStackable","nonStackableSeries","posBase","negBase","groupSpacing","exitTransition","last_datalength","barColor","rgb","darker","barSelection","multiBarChart","multibar","controlWidth","controlLabels","grouped","getTranslate","staggerUp","staggerDown","totalInBetweenTicks","reduceXTicks","multiBarHorizontal","valuePadding","getYerr","xerr","mid","path","yerr","showBarLabels","yErr","multiBarHorizontalChart","multiChart","mouseover_line","yaxis","yAxis2","yAxis1","mouseover_scatter","mouseover_stack","stack1","mouseover_bar","bars1","serieIndex","dataLines1","dataLines2","dat
aScatters1","dataScatters2","dataBars1","dataBars2","dataStack1","dataStack2","color_array","lines1","scatters1","scatters2","stack2","lines1Wrap","scatters1Wrap","bars1Wrap","stack1Wrap","scatters2Wrap","stack2Wrap","extraValue1","aVal","extraValue2","yScale1","yDomain1","yScale2","yDomain2","stackedArea","parallelCoordinates","enabledDimensions","displayMissingValuesline","axisWithUndefinedValues","newscale","missingValuesline","missingValueslineText","restoreBrush","visible","filters","brushDomain","dimension","hasOnlyNaN","oldDomainMaxValue","hasNaN","dimensions","updateTicks","brushstart","displayBrush","actives","dimensionNames","extents","foreground","isActive","brushend","hasActiveBrush","brushEnd","currentTicks","tickValues","dragging","__origin__","background","dimensionPosition","currentPosition","dimensionsOrder","newData","val","dataValues","dimensionData","rangePoints","onlyUndefinedValues","tension","lineTension","axisDrag","lineData","undefinedValuesLabel","dimension
sEnter","formerActive","activeChanged","dimensionFormats","parallelCoordinatesChart","originalPosition","parallelCoordinatesWrap","isSorted","nanValue","str","tp","dim","dd","pie","arcTween","endAngle","startAngle","donut","innerRadius","_current","arcs","arcsRadiusOuter","arcsRadiusInner","arcsRadius","outer","inner","donutRatio","growOnHover","g_pie","arcsOver","arc","outerRadius","arcOver","cornerRadius","padAngle","titleOffset","slices","pieLabels","ae","attrTween","showLabels","labelsArc","labelsOutside","group","labelSunbeamLayout","rotateAngle","centroid","labelLocationHash","avgHeight","avgWidth","createHashKey","coordinates","getSlicePercentage","center","labelThreshold","hashKey","labelType","pieLabelsOutside","donutLabelsOutside","labelFormat","pieChart","pieWrap","showTooltipPercent","sankey","computeNodeLinks","sourceLinks","targetLinks","computeNodeValues","sum","computeNodeBreadths","nextNodes","remainingNodes","nodeWidth","sinksRight","moveSinksRight","scaleNodeBread
ths","kx","computeNodeDepths","iterations","initializeNodeDepth","ky","nodesByBreadth","nodePadding","relaxLeftToRight","weightedSource","sy","breadth","relaxRightToLeft","weightedTarget","ty","resolveCollisions","ascendingDepth","nest","sortKeys","ascending","entries","computeLinkDepths","ascendingSourceDepth","ascendingTargetDepth","relayout","xi","interpolateNumber","curvature","x3","linkPath","sankeyChart","dragmove","testData","isDataValid","dataAvailable","error","showError","linkTitle","origin","appendChild","nodeFillColor","nodeStrokeColor","nodeTitle","units","formatNumber","message","nodeStyle","fillColor","strokeColor","getCache","_cache","delCache","getDiffs","cache","diffs","hasOwnProperty","updateInteractiveLayer","needsUpdate","vertices","groupIndex","pX","pY","pointArray","bounds","geom","polygon","voronoi","clip","pointPaths","vPointPaths","showVoronoi","clipVoronoi","pointClips","clipRadius","mouseEventCallback","el","mDispatch","scrollTop","pageYOffset","scrollLef
t","pageXOffset","relativePos","logScale","sizeDomain","getSize","padDataOuter","z","forceSize","sizeRange","_sizeRange_def","z0","scaleDiff","width0","height0","sizeDiff","pointBorderColor","points","getShape","titles","interactiveUpdateDelay","clearTimeout","timeoutID","shape","notActive","pointScale","pointRange","forcePoint","pointShape","scatterChart","showDistX","distX","showDistY","distY","regWrap","regLine","intercept","slope","sparkline","paths","result","yValues","maxPoint","lastIndexOf","minPoint","currentPoint","showMinMaxPoints","showCurrentPoint","animate","sparklinePlus","updateValueLine","paused","hoverValue","hoverEnter","sparklineHover","getClosestIndex","closestIndex","sparklineWrap","showLastValue","valueWrap","rightAlignValue","alignValue","dataRaw","aseries","dataFiltered","order","out","zeroArea","areaMouseover","pageX","pageY","areaMouseout","areaClick","d3_stackedOffset_stackPercent","stackData","o","stackedAreaChart","currentFormat","oldYTickFormat","percen
tFormatter","stackedWrap","xAxisHeight","legendTop","metaKey","stream","stack_percent","controlOptions","requiredTop","valueSum","allNullValues","tooltipValue","showTotalInTooltip","stackedY0","stackedY","totalLabel","oldValueFormatter","stackedAreaWithFocusChart","sunburst","rotationToAvoidUpsideDown","centerAngle","computeCenterAngle","computeNodePercentage","labelThresholdMatched","arcTweenZoom","xd","yd","yr","arcTweenUpdate","ipo","dx0","dy0","updatePrevPosition","prevPositions","pP","storeRetrievePrevPositions","zoomClick","depth","arcText","getBBox","rotation","partition","modes","mode","cG","cGE","groupColorByParent","children","parent","count","d1","d2","sunburstChart"],"mappings":";;AAAA,YAEA,GAAIA,KAGJA,GAAGC,KAAM,EACTD,EAAGE,QAAUF,EAAGE,YAChBF,EAAGG,MAAQH,EAAGG,UACdH,EAAGI,OAASJ,EAAGI,WACfJ,EAAGK,UACHL,EAAGM,QACHN,EAAGO,OAGoB,mBAAb,SAAgD,mBAAd,UAA2C,mBAAR,MAC3EC,GAAKC,QAAO,OAGhBT,EAAGU,SAAWF,GAAGE,SAAQ,eAAiB,cAOrCC,SAASC,UAAUC,OACpBF,SAASC,UAAUC,KAAO,SAAUC,GAChC,GAAoB,kB
AATC,MAEP,KAAM,IAAIC,WAAS,uEAGvB,IAAIC,GAAQC,MAAMN,UAAUO,MAAMC,KAAKC,UAAW,GAC9CC,EAAUP,KACVQ,EAAO,aACPC,EAAS,WACL,MAAOF,GAAQG,MAAMV,eAAgBQ,IAAQT,EACnCC,KACAD,EACNG,EAAMS,OAAOR,MAAMN,UAAUO,MAAMC,KAAKC,aAKpD,OAFAE,GAAKX,UAAYG,KAAKH,UACtBY,EAAOZ,UAAY,GAAIW,GAChBC,IAKXxB,EAAGC,MACHD,EAAGU,SAASiB,GAAE,eAAiB,SAASC,GACpC5B,EAAGM,KAAKuB,WAAa,GAAIC,QAG7B9B,EAAGU,SAASiB,GAAE,aAAe,SAASC,GAClC5B,EAAGM,KAAKyB,SAAW,GAAID,MACvB9B,EAAGM,KAAK0B,UAAYhC,EAAGM,KAAKyB,QAAU/B,EAAGM,KAAKuB,UAC9C7B,EAAGiC,IAAG,QAAUjC,EAAGM,KAAK0B,cAQhChC,EAAGiC,IAAM,WACL,GAAIjC,EAAGC,KAAOiC,OAAOC,SAAWA,QAAQF,KAAOE,QAAQF,IAAIR,MACvDU,QAAQF,IAAIR,MAAMU,QAASd,eAC1B,IAAIrB,EAAGC,KAAOiC,OAAOC,SAAiC,kBAAfA,SAAQF,KAAqBtB,SAASC,UAAUC,KAAM,CAC9F,GAAIoB,GAAMtB,SAASC,UAAUC,KAAKO,KAAKe,QAAQF,IAAKE,QACpDF,GAAIR,MAAMU,QAASd,WAEvB,MAAOA,WAAUA,UAAUe,OAAS,IAIxCpC,EAAGqC,WAAa,SAASC,EAAMC,GACvBJ,SAAWA,QAAQK,MACnBL,QAAQK,KAAI,kBAAqBF,EAAO,0BAA2BC,GAAQ,KAOnFvC,EAAGyC,OAAS,SAAgBC,GAExBA,EAAOA,GAAQ,EAEf1C,EAAGyC,OAAOE,QAAS,EACnB3C,EAAGU,SAASkC,c
AEZ,IAAIC,GAAa,WAGb,IAAK,GAFDC,GAAOC,EAEFC,EAAI,EAAON,EAAJM,IAAaD,EAAQ/C,EAAGyC,OAAOQ,MAAMD,IAAKA,IACtDF,EAAQC,EAAMG,iBACHH,GAAMI,gBAAkB,WAAYJ,EAAMI,SAASL,EAGlE9C,GAAGyC,OAAOQ,MAAMG,OAAO,EAAGJ,GAEtBhD,EAAGyC,OAAOQ,MAAMb,OAChBiB,WAAWR,IAGX7C,EAAGU,SAAS4C,aACZtD,EAAGyC,OAAOE,QAAS,GAI3BU,YAAWR,IAGf7C,EAAGyC,OAAOE,QAAS,EACnB3C,EAAGyC,OAAOQ,SAmBVjD,EAAGuD,SAAW,SAASC,SACRnC,WAAU,UAAa,YAC9BmC,GAAON,SAAU7B,UAAU,GAAI8B,SAAU9B,UAAU,KAGvDrB,EAAGyC,OAAOQ,MAAMQ,KAAKD,GAEhBxD,EAAGyC,OAAOE,QACX3C,EAAGyC,UAKY,mBAAb,SAAgD,mBAAd,WAC1CiB,OAAOC,QAAU3D,GAGI,mBAAb,UACRkC,OAAOlC,GAAKA,GClJdA,EAAGO,IAAIqD,MAAQ,SAAST,GACvB,MAAuBU,UAAnB3B,OAAO4B,QACHA,QAAQC,OAAOZ,GAEhBA,KASRnD,EAAGO,IAAIyD,KAAO,SAASb,GACtB,MAAuBU,UAAnB3B,OAAO4B,QACHA,QAAQG,QAAQd,GAEjBA,KCfRnD,EAAGkE,qBAAuB,WACtB,YAkBA,SAASC,GAAMC,GACXA,EAAUC,KAAK,SAASC,GAapB,QAASC,KACL,GAAIC,GAAUhE,GAAGiE,MAAM1D,MACnB2D,EAASF,EAAQ,GACjBG,EAASH,EAAQ,GACjBI,GAAiB,EACjBC,GAAoB,CAuCxB,IAtCIC,IAQAJ,EAASlE,GAAGuE,MAAMC,QAClBL,EAASnE,GAAGuE,MAAME,QAWa,QAA5BzE,GAAGu
E,MAAMG,OAAOC,UACfP,GAAiB,GAGjBpE,GAAGuE,MAAMG,OAAOE,UAAUC,QAAQC,MAAK,eACvCT,GAAoB,IAKzBD,IACCF,GAAUa,EAAOC,KACjBb,GAAUY,EAAOE,KAMC,aAAlBjF,GAAGuE,MAAMW,MACG,EAAThB,GAAuB,EAATC,GACdD,EAASiB,GAAkBhB,EAASiB,GACnCpF,GAAGuE,MAAMc,eAA4DhC,SAA3CrD,GAAGuE,MAAMc,cAAcC,iBAClDjB,EACD,CAEF,GAAIC,GACItE,GAAGuE,MAAMc,eACqChC,SAA3CrD,GAAGuE,MAAMc,cAAcC,kBACejC,SAArCrD,GAAGuE,MAAMc,cAAcT,WACpB5E,GAAGuE,MAAMc,cAAcT,UAAUE,MAAMpF,EAAQ6F,uBAEtD,MASR,OANArF,GAASsF,iBACLtB,OAAQA,EACRC,OAAQA,IAEZR,EAAM8B,gBAAgB,UACtB/F,GAAQgG,QAAO,GAGfhG,EAAQgG,QAAO,EAInB,IAAIC,GAA8C,kBAAtBC,GAAOC,WAC/BC,EAAczC,MAGlB,IAAIsC,EAAgB,CAChB,GAAII,GAAe/F,GAAGgG,OAAOJ,EAAOK,QAAS/B,GAAU,CAEvD,MAAI0B,EAAOK,QAAQF,GAAgBH,EAAOM,aAAehC,GAUrD,MANAhE,GAASsF,iBACLtB,OAAQA,EACRC,OAAQA,IAEZR,EAAM8B,gBAAgB,UACtB/F,GAAQgG,QAAO,EARfI,GAAcF,EAAOO,SAASnG,GAAGgG,OAAOJ,EAAOK,QAAS/B,GAAU,OAatE4B,GAAcF,EAAOQ,OAAOlC,EAGhChE,GAASmG,kBACLnC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAIK,aAAlB9F,GAAGuE,MAAMW,MACThF,EAASoG,iBACLpC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,I
AKC,UAAlB9F,GAAGuE,MAAMW,MACThF,EAASqG,cACLrC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAKC,cAAlB9F,GAAGuE,MAAMW,MACZhF,EAASsG,kBACRtC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAKO,YAAlB9F,GAAGuE,MAAMW,MACZhF,EAASuG,gBACRvC,OAAQA,EACRC,OAAQA,EACR2B,YAAaA,IAlJnB,GAAIY,GAAY1G,GAAG2G,OAAOpG,MACtB4E,EAAkByB,GAAS,IAAMxB,EAAmByB,GAAU,IAC9DC,EAAOJ,EAAUK,UAAS,qCACzBjD,MAAMA,IACPkD,EAAYF,EAAKG,QAChBC,OAAM,KAAMC,KAAI,QAAU,mCAC/BH,GAAUE,OAAM,KAAMC,KAAI,QAAO,2BAE5BC,IA+ILA,EACKjG,GAAE,YAAa4C,GACf5C,GAAE,YAAa4C,GAAc,GAC7B5C,GAAE,WAAa4C,GAAa,GAC5B5C,GAAE,YAAc4C,GAAa,GAC7B5C,GAAE,UAAY4C,GAAa,GAC3B5C,GAAE,WAAa4C,GACf5C,GAAE,QAAU4C,GAGjBJ,EAAM0D,UAAY,KAElB1D,EAAM8B,gBAAkB,SAAS6B,GACxBC,IACD5D,EAAM0D,WAAa1D,EAAM0D,UAAUF,KAAI,QAAWG,GACtD9H,EAAGO,IAAIqD,MAAM,WACT,GAAIoE,GAAOV,EAAKH,OAAM,4BACjBI,UAAS,QACTjD,KAAW,MAALwD,GAAc9H,EAAGG,MAAM8H,UAAUH,OAAUI,OACtDF,GAAKP,QACAC,OAAM,QACNC,KAAI,QAAU,gBACdA,KAAI,KAAO,SAASQ,GAAK,MAAOA,KAChCR,KAAI,KAAO,SAASQ,GAAK,MAAOA,KAChCR,KAAI,KAAO/B,GACX+B,KAAI,KAAM,GACfK,EAAKI,OAAOC,gBAnM5B,GAAI9
C,IAAWC,KAAM,EAAGC,IAAK,GACrB2B,EAAQ,KACRC,EAAS,KACTjB,EAAS5F,GAAG8H,MAAMC,SAClB7H,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,eAAgB,kBAAmB,mBAAoB,kBACrHqH,GAAgB,EAChBH,EAAe,KACf1H,EAAUF,EAAGI,OAAOF,UACpB4E,EAAU5C,OAAOsG,aAyOzB,OAtOAtI,GACKuI,SAAS,GACTC,UAAU,GACVxC,QAAO,GA2LZ/B,EAAMzD,SAAWA,EACjByD,EAAMjE,QAAUA,EAEhBiE,EAAMoB,OAAS,SAASoD,GACpB,MAAKtH,WAAUe,QACfmD,EAAOE,IAA4B,mBAAZkD,GAAElD,IAAwBkD,EAAElD,IAASF,EAAOE,IACnEF,EAAOC,KAA4B,mBAAZmD,GAAEnD,KAAwBmD,EAAEnD,KAASD,EAAOC,KAC5DrB,GAHuBoB,GAMlCpB,EAAMiD,MAAQ,SAASuB,GACnB,MAAKtH,WAAUe,QACfgF,EAAQuB,EACDxE,GAFuBiD,GAKlCjD,EAAMkD,OAAS,SAASsB,GACpB,MAAKtH,WAAUe,QACfiF,EAASsB,EACFxE,GAFuBkD,GAKlClD,EAAMiC,OAAS,SAASuC,GACpB,MAAKtH,WAAUe,QACfgE,EAASuC,EACFxE,GAFuBiC,GAKlCjC,EAAM4D,cAAgB,SAASY,GAC3B,MAAKtH,WAAUe,QACf2F,EAAgBY,EACTxE,GAFuB4D,GAKlC5D,EAAMyD,aAAe,SAASe,GAC1B,MAAKtH,WAAUe,QACfwF,EAAee,EACRxE,GAFuByD,GAK3BzD,GAgBXnE,EAAG4I,kBAAoB,SAAUC,EAAQC,EAAWC,GAChD,YACA,MAAOF,YAAkB3H,QACrB,MAAO,KAEX,IAAI8H,EAEAA,GADqB,kBAAdD,GACM,SAASZ,GAClB,MAAOA,GA
AEL,GAGAiB,CAEjB,IAAIE,GAAO,SAASd,EAAGe,GAUnB,MAAOF,GAAWb,GAAKe,GAGvB1C,EAAShG,GAAG2I,SAASF,GAAMzD,KAC3B4D,EAAQ5I,GAAG6I,KAAK,EAAG7C,EAAOqC,EAAOC,GAAa,IAC9CQ,EAAeN,EAAWH,EAAOO,GAMrC,IAJ4B,mBAAjBE,KACPA,EAAeF,GAGfE,IAAiBR,EACjB,MAAOM,EAGX,IAAIG,GAAY/I,GAAGgJ,KAAKJ,EAAM,EAAGP,EAAOzG,OAAS,IAC7CqH,EAAYT,EAAWH,EAAOU,GAMlC,OAJyB,mBAAdE,KACPA,EAAYF,GAGZG,KAAKC,IAAIF,EAAYX,IAAcY,KAAKC,IAAIL,EAAeR,GACpDM,EAEAG,GASfvJ,EAAG4J,kBAAoB,SAAUf,EAAQC,EAAWe,GAChD,YACA,IAAIC,GAAWC,EAAAA,EAAUC,EAAmB,IAQ5C,OAPAnB,GAAOoB,QAAQ,SAAS9B,EAAEnF,GACtB,GAAIkH,GAAQR,KAAKC,IAAIb,EAAYX,EACvB,OAALA,GAAsB2B,GAATI,GAA6BL,EAARK,IACnCJ,EAAWI,EACXF,EAAmBhH,KAGpBgH,GCvUXhK,EAAGI,OAAOF,QAAU,WAChB,YAyPA,SAASiK,KACL,IAAKjK,IAAYA,EAAQkK,OAAQ,CAG7B,GAAI9F,IAAQ,EACZpE,GAAUM,GAAG2G,OAAOkD,SAASC,MAAMnD,OAAM,IAAKoD,GAAIjG,KAAKA,GAEvDpE,EAAQuH,QAAQC,OAAM,OACdC,KAAI,QAAU,cAAgB6C,EAAUA,EAAU,eAClD7C,KAAI,KAAO4C,GACXE,MAAK,MAAQ,GAAGA,MAAK,OAAS,GAC9BA,MAAK,UAAY,GACjBA,MAAK,WAAa,SAClBlD,UAAS,sBAAuBmD,QAAQ3E,GAAsB,GAC9D2E,QAAQ3E,GAAsB,GA
EtC7F,EAAQkI,OAAOC,UAKvB,QAASsC,KACL,MAAKC,IACAC,EAAiBvG,IAEtBtE,EAAGO,IAAIqD,MAAM,WACTuG,GAIA,IAAIW,GAAaC,EAAiBzG,EAC9BwG,KACA5K,EAAQkK,OAAOY,UAAYF,GAG/BG,MAGGN,GAhBP,OAjQJ,GAAIJ,GAAK,aAAeb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UAChC7G,EAAO,KACP8G,EAAU,IACVC,EAAW,GACXC,EAAe,EACfd,EAAU,KACVtE,GAAS,EACTwC,EAAY,IACZxI,EAAU,KACVqL,GAAiB/F,KAAM,KAAMC,IAAK,MAClCmF,GAAU,EACVnC,EAAW,IACX+C,GAAgB,EAChBzF,EAAuB,yBAI3B0F,EAAiB,SAAStD,EAAGnF,GAC7B,MAAOmF,IAIPuD,EAAkB,SAASvD,GAC3B,MAAOA,IAGPwD,EAAe,SAASxD,EAAGnF,GAC3B,MAAOmF,IAKP4C,EAAmB,SAAS5C,GAC5B,GAAU,OAANA,EACA,MAAO,EAGX,IAAIyD,GAAQpL,GAAG2G,OAAOkD,SAASwB,cAAa,SAC5C,IAAIL,EAAe,CACf,GAAIM,GAAaF,EAAMrE,UAAS,SAC3BjD,MAAM6D,IACNV,QAAQC,OAAM,QAEnBoE,GAAWpE,OAAM,MACZA,OAAM,MACNC,KAAI,UAAY,GAChBD,OAAM,UACNgD,QAAO,WAAY,GACnBqB,KAAKL,EAAgBvD,EAAE6D,QAGhC,GAAIC,GAAaL,EAAMrE,UAAS,SAC3BjD,MAAM6D,IACNV,QAAQC,OAAM,SAEfwE,EAAYD,EAAW1E,UAAS,MAC3BjD,KAAK,SAAS6H,GAAK,MAAOA,GAAEC,SAC5B3E,QACAC,OAAM,MACNgD,QAAO,YAAc,SAASyB,GAAK,MAAOA,GAAEE,WAErDH,GAAUxE,OAAM,MACXgD,QAA
O,sBAAsB,GAC7BhD,OAAM,OACN+C,MAAK,mBAAqB,SAAS0B,GAAK,MAAOA,GAAEG,QAEtDJ,EAAUxE,OAAM,MACXgD,QAAO,OAAO,GACdA,QAAO,QAAS,SAASyB,GAAK,QAASA,EAAEI,QACzCR,KAAK,SAASI,EAAGnJ,GAAK,MAAO2I,GAAaQ,EAAEK,IAAKxJ,KAEtDkJ,EAAUxE,OAAM,MACXgD,QAAO,SAAS,GAChBqB,KAAK,SAASI,EAAGnJ,GAAK,MAAOyI,GAAeU,EAAEH,MAAOhJ,KAE1DkJ,EAAUO,OAAO,SAAUN,EAAEnJ,GAAK,MAAqBa,UAAdsI,EAAEO,UAAyBhF,OAAM,MACrEgD,QAAO,WAAY,GACnBqB,KAAK,SAASI,EAAGnJ,GAAK,MAAO,IAAMxC,GAAGmM,OAAM,KAAMR,EAAEO,SAAW,MAEpER,EAAU3E,UAAS,MAAOlD,KAAK,SAAS8H,GACpC,GAAIA,EAAEE,UAAW,CACb,GAAIO,GAAepM,GAAG8H,MAAMC,SAAS5B,QAAQ,EAAE,IAAIF,OAAK,OAAS0F,EAAEG,QAC/DO,EAAU,EACdrM,IAAG2G,OAAOpG,MACL0J,MAAK,sBAAwBmC,EAAaC,IAC1CpC,MAAK,mBAAqBmC,EAAaC,MAKpD,IAAId,GAAOH,EAAMxB,OAAO0C,SAGxB,OAFiBjJ,UAAbsE,EAAE4E,SACFhB,GAAQ,uBAAyB5D,EAAE4E,OAAS,UACzChB,GAYPiB,EAAW,WACX,GAAIC,IACAzH,KAAmB,OAAbhF,GAAGuE,MAAiBvE,GAAGuE,MAAMmI,QAAU,EAC7CzH,IAAkB,OAAbjF,GAAGuE,MAAiBvE,GAAGuE,MAAMoI,QAAU,EAGhD,IAAgD,QAA7CC,iBAAiB/C,SAASC,MAAM+C,UAAqB,CAGpD,GAAIC,GAASjD,SAASC,KAAKiD,uBAC3BN,GAAIz
H,MAAQ8H,EAAO9H,KACnByH,EAAIxH,KAAO6H,EAAO7H,IAGtB,MAAOwH,IAGPpC,EAAmB,SAAS1C,GAC5B,GAAIA,GAAKA,EAAEiE,OAAQ,CACf,GAAIpM,EAAGG,MAAMqN,QAAQrF,EAAEiE,QACnB,OAAO,CAGX,IAAIpM,EAAGG,MAAMsN,SAAStF,EAAEiE,QAEpB,MADAjE,GAAEiE,QAAUjE,EAAEiE,SACP,EAGf,OAAO,GAKPsB,EAAoB,SAAST,GAC7B,GAIIzH,GAAMC,EAAKkI,EAJXtG,EAASnH,EAAQkK,OAAOwD,aACxBxG,EAAQlH,EAAQkK,OAAOyD,YACvBC,EAAczD,SAAS0D,gBAAgBD,YACvCE,EAAe3D,SAAS0D,gBAAgBC,YAI5C,QAAQ5C,GACJ,IAAK,IACD5F,GAAS4B,EAAQiE,EACjB5F,IAAS4B,EAAS,GACf4F,EAAIzH,KAAOA,EAAO,IAAGA,EAAO6F,IAC3BsC,EAAMV,EAAIxH,IAAMA,GAAO,IAAGA,GAAOkI,IACjCA,EAAMV,EAAIxH,IAAMA,EAAM4B,GAAU2G,IAAcvI,GAAOkI,EAAMK,EAC/D,MACJ,KAAK,IACDxI,EAAO6F,EACP5F,IAAS4B,EAAS,GACd4F,EAAIzH,KAAOA,EAAO4B,EAAQ0G,IAAatI,GAAS4B,EAAQiE,IACvDsC,EAAMV,EAAIxH,IAAMA,GAAO,IAAGA,GAAOkI,IACjCA,EAAMV,EAAIxH,IAAMA,EAAM4B,GAAU2G,IAAcvI,GAAOkI,EAAMK,EAChE,MACJ,KAAK,IACDxI,IAAU4B,EAAQ,GAAK,EACvB3B,EAAM4F,EACF4B,EAAIxH,IAAMA,EAAM4B,EAAS2G,IAAcvI,GAAQ4B,EAASgE,IACvDsC,EAAMV,EAAIzH,KAAOA,GAAQ,IAAGA,GAAQmI,IACpCA,EAAMV,EAAIzH,
KAAOA,EAAO4B,GAAS0G,IAAatI,GAAQmI,EAAMG,EACjE,MACJ,KAAK,IACDtI,IAAU4B,EAAQ,GAClB3B,GAAQ4B,EAASgE,EACb4B,EAAIxH,IAAMA,EAAM,IAAGA,EAAM4F,IACxBsC,EAAMV,EAAIzH,KAAOA,GAAQ,IAAGA,GAAQmI,IACpCA,EAAMV,EAAIzH,KAAOA,EAAO4B,GAAS0G,IAAatI,GAAQmI,EAAMG,EACjE,MACJ,KAAK,SACDtI,IAAU4B,EAAQ,GAClB3B,IAAS4B,EAAS,EAClB,MACJ,SACI7B,EAAO,EACPC,EAAM,EAId,OAASD,KAAQA,EAAMC,IAAOA,IAM9BwF,EAAkB,WAClBjL,EAAGO,IAAIyD,KAAK,WACR,GAAIiJ,GAAMD,IACNiB,EAAgBP,EAAkBT,GAClCzH,EAAOyH,EAAIzH,KAAOyI,EAAczI,KAChCC,EAAMwH,EAAIxH,IAAMwI,EAAcxI,GAGlC,IAAIS,EACAhG,EACKgO,YACAC,aACAC,MAAM1F,GACND,SAAS,GACTgC,MAAK,UAAY,OACnB,CAEH,GAAI4D,GAAgB,aAAe9C,EAAa/F,KAAO,OAAS+F,EAAa9F,IAAM,MAC/E6I,EAAgB,aAAe5E,KAAK6E,MAAM/I,GAAQ,OAASkE,KAAK6E,MAAM9I,GAAO,MAC7E+I,EAAwBhO,GAAGiO,kBAAkBJ,EAAeC,GAC5DI,EAAYxO,EAAQuK,MAAK,WAAc,EAE3CvK,GACKgO,YACAC,aACA1F,SAASiG,EAAY,EAAIjG,GAEzBkG,WAAU,YAAc,SAAUxG,GAC/B,MAAOqG,IACR,aAEFG,WAAU,oBAAsB,SAAUxG,GACvC,MAAOqG,KAEV/D,MAAK,gBAAkB6D,GACvB7D,MAAK,UAAY,GAG1Bc,EAAa/F,KAAOA,EACpB+F,EAAa9F,IAAMA,IAuG3B,OAzDA
kF,GAAU5E,qBAAuBA,EACjC4E,EAAUiE,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAK8J,GAE9CA,EAAUmE,SAAWC,OAAOC,WAExBvG,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAAGF,EAASE,IACxEyC,SAAU6D,IAAK,WAAW,MAAO7D,IAAW8D,IAAK,SAASvG,GAAGyC,EAAQzC,IACrE0C,UAAW4D,IAAK,WAAW,MAAO5D,IAAY6D,IAAK,SAASvG,GAAG0C,EAAS1C,IACxE2C,cAAe2D,IAAK,WAAW,MAAO3D,IAAgB4D,IAAK,SAASvG,GAAG2C,EAAa3C,IACpF6B,SAAUyE,IAAK,WAAW,MAAOzE,IAAW0E,IAAK,SAASvG,GAAG6B,EAAQ7B,IACrEiC,SAAUqE,IAAK,WAAW,MAAOrE,IAAWsE,IAAK,SAASvG,GAAGiC,EAAQjC,IACrED,WAAYuG,IAAK,WAAW,MAAOvG,IAAawG,IAAK,SAASvG,GAAGD,EAAUC,IAC3EoC,kBAAmBkE,IAAK,WAAW,MAAOlE,IAAoBmE,IAAK,SAASvG,GAAGoC,EAAiBpC,IAChG8C,gBAAiBwD,IAAK,WAAW,MAAOxD,IAAkByD,IAAK,SAASvG,GAAG8C,EAAe9C,IAC1F+C,iBAAkBuD,IAAK,WAAW,MAAOvD,IAAmBwD,IAAK,SAASvG,GAAG+C,EAAgB/C,IAC7FgD,cAAesD,IAAK,WAAW,MAAOtD,IAAgBuD,IAAK,SAASvG,GAAGgD,EAAahD,IACpF6C,eAAgByD,IAAK,WAAW,MAAOzD,IAAiB0D,IAAK,SAASvG,GAAG6C,EAAc7C,IACvFqE,UAAWiC,IAAK,WAAW,MAAOjC,IAAYkC,IAAK,SAASvG,GAAGqE,EAASrE,IAGxEwG,gBAAiBF,IAAK,WAAW,MAAO5E,UAASC,MAAQ4E,IAA
K,SAASvG,GAEnE3I,EAAGqC,WAAU,iBAAmB,iCAEpC+M,UAAWH,IAAK,WAAW,MAAO,OAAQC,IAAK,SAASvG,GAEpD3I,EAAGqC,WAAU,WAAa,iCAE9BgN,QAASJ,IAAK,WAAW,OAAQzJ,KAAM,EAAGC,IAAK,IAAMyJ,IAAK,SAASvG,GAE/D3I,EAAGqC,WAAU,SAAW,0CAI5B6D,QAAS+I,IAAK,WAAW,MAAO/I,IAAUgJ,IAAK,SAASvG,GAChDzC,GAAUyC,IACVzC,IAAWyC,EACXgC,OAGRrG,MAAO2K,IAAK,WAAW,MAAO3K,IAAQ4K,IAAK,SAASvG,GAE5CA,EAAE2G,QACF3G,EAAEqD,MAAQrD,EAAE2G,MAAMxH,EAClBa,EAAEyD,OAASzD,EAAEyD,WACbzD,EAAEyD,OAAOJ,MAAQrD,EAAE2G,MAAMC,EACzB5G,EAAEyD,OAAOE,MAAQ3D,EAAE2G,MAAMhD,OAAS3D,EAAEyD,OAAOE,OAE/ChI,EAAOqE,IAIXyB,MAAO6E,IAAK,WAAW,MAAO/O,GAAQkK,QAAU8E,IAAK,SAASvG,MAC9D4B,IAAK0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,QAGpD3I,EAAGG,MAAMqP,YAAY7E,GACdA,GC7VX3K,EAAGG,MAAMsP,WAAa,WAElB,GAAIC,IAAQtI,MAAO,IAAKC,OAAQ,IAGhC,OAAInF,QAAOyN,YAAczN,OAAO0N,aAC5BF,EAAKtI,MAAQlF,OAAOyN,WACpBD,EAAKrI,OAASnF,OAAO0N,YACd,GAIY,cAAnBvF,SAASwF,YACTxF,SAAS0D,iBACT1D,SAAS0D,gBAAgBF,aAEzB6B,EAAKtI,MAAQiD,SAAS0D,gBAAgBF,YACtC6B,EAAKrI,OAASgD,SAAS0D,gBAAgBH,aAChC,GAIPvD,SAASC,MAAQD,SAASC,KAAK
uD,aAC/B6B,EAAKtI,MAAQiD,SAASC,KAAKuD,YAC3B6B,EAAKrI,OAASgD,SAASC,KAAKsD,aACrB,GAGJ,GAOX5N,EAAGG,MAAMqN,QAAUtM,MAAMsM,QACzBxN,EAAGG,MAAMsN,SAAW,SAASqC,GACzB,MAAa,QAANA,GAA2B,gBAANA,IAEhC9P,EAAGG,MAAM4P,WAAa,SAASD,GAC3B,MAAoB,kBAANA,IAElB9P,EAAGG,MAAM6P,OAAS,SAASF,GACvB,MAA4B,kBAArBG,SAAS7O,KAAK0O,IAEzB9P,EAAGG,MAAM+P,SAAW,SAASJ,GACzB,OAAQK,MAAML,IAAmB,gBAANA,IAO/B9P,EAAGG,MAAMiQ,aAAe,SAASC,GAO7B,MANInO,QAAOoO,iBACPpO,OAAOoO,iBAAgB,SAAWD,GAElCrQ,EAAGiC,IAAG,gDAAkDoO,IAIxDlN,SAAUkN,EACVE,MAAO,WACHrO,OAAOsO,oBAAmB,SAAWH,MAYjDrQ,EAAGG,MAAMsQ,SAAW,SAASnE,GAEzB,GAAczI,SAAVyI,EACA,MAAOtM,GAAGG,MAAMuQ,cAGb,IAAG1Q,EAAGG,MAAMqN,QAAQlB,GAAQ,CAC/B,GAAIqE,GAAcnQ,GAAG8H,MAAMsI,UAAUnK,MAAM6F,EAC3C,OAAO,UAASnE,EAAGnF,GACf,GAAIwJ,GAAY3I,SAANb,EAAkBmF,EAAInF,CAChC,OAAOmF,GAAEmE,OAASqE,EAAYnE,IAOlC,MAAOF,IASftM,EAAGG,MAAMuQ,aAAe,WAEpB,MAAO1Q,GAAGG,MAAMsQ,SAASjQ,GAAG8H,MAAMuI,aAAapK,UAQnDzG,EAAGG,MAAM2Q,YAAc,SAASC,EAAYC,EAAQC,GAEhDD,EAASA,GAAU,SAAS5E,GAAU,MAAOA,GAAOI,KACpDyE,EAAgBA,GAAiBzQ,GAAG8H,MAAMu
I,aAAapK,OAGvD,IAAIyK,GAAWD,EAAc7O,MAE7B,OAAO,UAASgK,EAAQhD,GACpB,GAAIoD,GAAMwE,EAAO5E,EACjB,OAAIpM,GAAGG,MAAM4P,WAAWgB,EAAWvE,IACxBuE,EAAWvE,KACS3I,SAApBkN,EAAWvE,GACXuE,EAAWvE,IAGb0E,IAEDA,EAAWD,EAAc7O,QAE7B8O,GAAsB,EACfD,EAAcC,MAWjClR,EAAGG,MAAMgR,KAAO,SAASC,EAAOC,GAE5B,GAAIC,GAAO,SAASC,GAChB/Q,GAAGuL,KAAKwF,EAAM,SAASC,GACnB,GAAItM,GAAS1E,GAAG2G,OAAOkK,GAASjH,MAChClF,GAAOuM,WAAWC,aACdlR,GAAG2G,OAAOqK,GAAUrK,OAAOkK,GAASjH,OACpClF,GACJlF,EAAGG,MAAMgR,KAAKC,EAAOC,KAI7B7Q,IAAG+G,UAAU6J,GAAOzP,GAAE,QAAU,WAC5BgQ,QAAQC,UAAU7Q,KAAKwQ,KAAMxQ,KAAK8Q,YAAa9Q,KAAKwQ,MACpDD,EAAKvQ,KAAKwQ,MACV/Q,GAAGuE,MAAM+M,mBAGbtR,GAAG2G,OAAOjF,QAAQP,GAAE,WAAa,WACzBnB,GAAGuE,MAAMgN,OACTT,EAAK9Q,GAAGuE,MAAMgN,UAW1B/R,EAAGG,MAAM6R,oBAAsB,SAAUC,GACrC,GAAIjS,EAAGG,MAAM4P,WAAWkC,EAAYxH,QAAUzK,EAAGG,MAAM4P,WAAWkC,EAAYC,MAAO,CACjF,GAAIC,GAAWC,SAASH,EAAYxH,MAAK,aAAc4H,QAAO,KAAI,IAAO,IACrEC,EAAaL,EAAYC,OAAO9P,MACpC,OAAOpC,GAAGG,MAAM8H,UAAUqK,EAAaH,EAAW,IAEtD,MAAO,IAOXnS,EAAGG,MAAM8H,UAAY,SAASsK,GAC1B,OAAKvS,EAAGG,MAA
M+P,SAASqC,IAChBpC,MAAMoC,IACA,OAANA,GACAA,IAAMxI,EAAAA,GACNwI,MAAOxI,EAAAA,GAEH,EAEJwI,GAMX/R,GAAG4D,UAAUxD,UAAU4R,gBAAkB,SAASC,GAC9C,GAAIC,IAAQ3R,MAAMW,UAAUP,MAAMC,KAAKC,UAAW,GAClD,OAAOoR,GAAYtE,WAAW1M,MAAMgR,EAAaC,IAOrD1S,EAAGG,MAAMsS,YAAc,SAAS/R,EAAU+H,GACtC,KAAM1H,eAAgBf,GAAGG,MAAMsS,aAC3B,MAAO,IAAIzS,GAAGG,MAAMsS,YAAY/R,EAAU+H,EAG9C,IAAIkK,GAAyB9O,SAAb4E,EAAyBA,EAAW,IAChDmK,KACAC,EAAO9R,IAEXA,MAAKX,OAAS,SAASA,GAevB,MAdIA,MAAYe,MAAMC,KAAKC,UAAW,GAClCjB,EAAO6J,QAAQ,SAAS6I,GACpBA,EAAMC,YAAa,EACnB,SAAUC,GACNA,EAAEtS,SAASiB,GAAE,YAAc,SAASsR,GAChCD,EAAED,YAAa,EACfF,EAAKK,UAAS,YAEnBJ,GAECF,EAAYO,QAAQL,GAAS,GAC7BF,EAAYnP,KAAKqP,KAGtB/R,MAGPA,KAAKqS,MAAQ,SAAS3K,GACD5E,SAAb4E,IACAkK,EAAYlK,GAEhBmK,MAGJ7R,KAAKoN,WAAa,SAAS/J,EAAWsO,EAAMjK,GAcxC,GAbAiK,EAAOrR,UAAUe,OAAS,KAAOjB,MAAMC,KAAKC,UAAW,MAGnDoH,EADAiK,EAAKtQ,OAAS,EACHsQ,EAAKW,MAESxP,SAAd8O,EAA0BA,EAAY,IAErDvO,EAAU2O,YAAa,EAEnBH,EAAYO,QAAQ/O,GAAa,GACjCwO,EAAYnP,KAAKW,GAGJ,IAAbqE,EAIA,MAHArE,GAAU2O,YAAa,EACvB3O,EAAUgK,MAAQ,WAAa,MAAOrN
,OACtCqD,EAAUqE,SAAW,WAAa,MAAO1H,OAClCqD,CAEkB,KAArBA,EAAUhC,OACVgC,EAAU2O,YAAa,EAChB3O,EAAUkP,MAAO,SAASnL,GAAI,OAAQA,EAAE/F,SAC/CgC,EAAU2O,YAAa,EAEvB3O,EAAU2O,YAAa,CAG3B,IAAIR,GAAI,CACR,OAAOnO,GACF+J,aACA1F,SAASA,GACTpE,KAAK,aAAckO,IACnBlO,KAAI,MAAQ,SAAS8D,EAAGnF,GACT,MAANuP,IACFnO,EAAU2O,YAAa,EACvBF,EAAKK,UAAUzR,MAAMV,KAAM2R,OAM/C3R,KAAKmS,UAAY,WACTN,EAAYU,MAAO,SAASnL,GAAI,MAAOA,GAAE4K,eACzCH,EAAY3I,QAAS,SAAS9B,GAAIA,EAAE4K,YAAa,IACjDrS,EAASwS,UAAUzR,MAAMV,KAAMM,cAY3CrB,EAAGG,MAAMoT,WAAa,SAASC,GAC3B,GAAIC,GAAUpS,UAAUe,OAAS,KAAOjB,MAAMC,KAAKC,UAAW,KAC9DoS,GAAQxJ,QAAQ,SAASyJ,GACrB,IAAK,GAAIlH,KAAOkH,GAAQ,CACpB,GAAIlG,GAAUxN,EAAGG,MAAMqN,QAAQgG,EAAIhH,IAC/BiB,EAAWzN,EAAGG,MAAMsN,SAAS+F,EAAIhH,IACjCmH,EAAS3T,EAAGG,MAAMsN,SAASiG,EAAOlH,GAElCiB,KAAaD,GAAWmG,EACxB3T,EAAGG,MAAMoT,WAAWC,EAAIhH,GAAMkH,EAAOlH,IAErCgH,EAAIhH,GAAOkH,EAAOlH,OAUlCxM,EAAGG,MAAM4R,MAAQ,WACb,KAAMhR,eAAgBf,GAAGG,MAAM4R,OAC3B,MAAO,IAAI/R,GAAGG,MAAM4R,KAExB,IAAIA,MAEA6B,EAAY,aACZC,EAAY,WAAY,UACxBC,EAAO,KACPC,EAAU,IAEd
hT,MAAKL,SAAWF,GAAGE,SAAQ,SAAW,OAEtCK,KAAKL,SAASiB,GAAE,MAAQ,SAASoQ,GAC7B6B,EAAU7B,GAAO,KAGrBhR,KAAKiT,OAAS,SAASC,GAEnB,MADAJ,GAAYI,EACLlT,MAGXA,KAAKmT,OAAS,SAASD,EAAI9Q,GAUvB,MATKA,KACDA,EAAW,cAEfyQ,EAAY,SAAS7B,EAAOoC,GACxBF,EAAGlC,GACCoC,GACAhR,KAGDpC,MAGXA,KAAK+S,KAAO,SAAS/B,GACjB+B,EAAOA,MACP9T,EAAGG,MAAMoT,WAAWO,EAAM/B,GAG9B,IAAIqC,GAAO,WACP,GAAIC,GAAWR,GAEf,IAAIS,KAAKC,UAAUF,KAAcC,KAAKC,UAAUxC,GAC5C,OAAO,CAGX,KAAK,GAAIvF,KAAO6H,GACOxQ,SAAfkO,EAAMvF,KACNuF,EAAMvF,OAEVuF,EAAMvF,GAAO6H,EAAS7H,GACtBuH,GAAU,CAEd,QAAO,EAGXhT,MAAKoT,OAAS,WACNL,IACAF,EAAUE,GAAM,GAChBA,EAAO,MAEPM,EAAKhT,KAAKL,OACVA,KAAKL,SAAS8T,OAAOzC,KAkBjC/R,EAAGG,MAAM0O,YAAc,SAAS6D,GAQ5B,MAPIA,IACAlS,GAAGiU,IAAI/B,GAAMzI,QAAO,SAAWuC,EAAIR,GAC3BhM,EAAGG,MAAM4P,WAAWhP,KAAKyL,KACzBzL,KAAKyL,GAAKR,IAEfnL,KAAKE,OAELA,MAWXf,EAAGG,MAAMuU,WAAa,SAASC,EAAUrQ,GAErC,GAAIsQ,GAAY,EACZ5R,EAAI,CACR,KAAKA,EAAGA,EAAIsB,EAAKlC,OAAQY,GAAK,EAAG,CAC7B,GAAI6R,GAAavQ,EAAKtB,IAAMsB,EAAKtB,GAAG6F,OAASvE,EAAKtB,GAAG6F,OAAOzG,OAAS,CACrEwS,GAAY
C,EAAaD,EAAYC,EAAaD,EAWtD,MATA5U,GAAGiC,IAAG,8BAAgC0S,GACtC3U,EAAGiC,IAAG,gCAAkC2S,GAExCD,EAAWA,EAAWC,EAAYD,EAAWC,EAAY,EAAID,EAE7DA,EAAsB,EAAXA,EAAe,EAAIA,EAE9BA,EAAWjL,KAAKwB,MAAMyJ,GACtB3U,EAAGiC,IAAG,8BAAgC0S,GAC/BA,GAOX3U,EAAGG,MAAM2U,WAAa,SAASH,EAAUrQ,GAErC,MAAOtE,GAAGG,MAAMuU,WAAWC,EAAUrQ,IAYzCtE,EAAGG,MAAM4U,WAAa,SAASjS,EAAOR,GAE9BQ,EAAMkS,QAAUlS,EAAMkS,OAAO1S,GAC7BQ,EAAMR,GAAQQ,EAAMkS,OAAO1S,IAE3BQ,EAAMR,GAAQ,SAAUqG,GACpB,MAAKtH,WAAUe,QACfU,EAAMmS,WAAW3S,IAAQ,EACzBQ,EAAMgM,SAASxM,GAAQqG,EAChB7F,GAHuBA,EAAMgM,SAASxM,IAOjDQ,EAAK,IAAOR,GAAQ,SAASqG,GACzB,MAAKtH,WAAUe,QACVU,EAAMmS,WAAW3S,KAClBQ,EAAMgM,SAASxM,GAAQqG,GAEpB7F,GAJuBA,EAAMgM,SAASxM,MAazDtC,EAAGG,MAAMqP,YAAc,SAAS1M,GAC5BA,EAAMmS,WAAanS,EAAMmS,cACzB,IAAIC,GAAMnG,OAAOoG,oBAAoBrS,EAAMgM,cACvCsG,EAAQrG,OAAOoG,oBAAoBrS,EAAMkS,WAC7CE,GAAMA,EAAIxT,OAAO0T,EACjB,KAAK,GAAIpS,KAAKkS,GACVlV,EAAGG,MAAM4U,WAAWjS,EAAOoS,EAAIlS,KAUvChD,EAAGG,MAAMkV,iBAAmB,SAASnQ,EAAQoQ,EAAWC,GACpDrQ,EAAOsQ,WAAaD,EAAO7T,OAAOwD,EAAOsQ,gBACzCD,EAAOE,QAAQ
H,GACfC,EAAOE,QAAQvQ,GACf1E,GAAGkV,OAAOjU,MAAMV,KAAMwU,IAO1BvV,EAAGG,MAAMwV,YAAc,SAAS7F,GAC5B,MAAOA,GAAE8F,OAAOnJ,OAAO,SAASoJ,EAAM5I,GAClC,OAAQA,GAAO4I,GAAQ/F,EAAE7C,EAAM,MAUvCjN,EAAGG,MAAM2V,UAAYtV,GAAGiU,MAMxBzU,EAAGG,MAAM4V,OAAS,WAGd,QAASA,GAAO5N,EAAEnF,GACd,GAAIgT,GAAItQ,EAAKtE,KAAKL,KAAKoH,EAAEnF,GACrBiT,EAAIvG,EAAKtO,KAAKL,KAAKoH,EAAEnF,EACzB,OAAsC,KAAlCxC,GAAG0V,IAAIC,YAAYhD,QAAQ6C,GACpBxV,GAAG0V,IAAIH,SAASrQ,KAAKsQ,GAAGtG,KAAKuG,KAE7BjW,EAAGG,MAAM2V,UAAU7G,IAAI+G,GAAGC,GARzC,GAAIvQ,GACAgK,EAAO,EAoBX,OAVAqG,GAAOrQ,KAAO,SAASiD,GACnB,MAAKtH,WAAUe,QACfsD,EAAOlF,GAAG4V,QAAQzN,GACXoN,GAFuBrQ,GAIlCqQ,EAAOrG,KAAO,SAAS/G,GACnB,MAAKtH,WAAUe,QACfsN,EAAOlP,GAAG4V,QAAQzN,GACXoN,GAFuBrG,GAI3BqG,GAUX/V,EAAGG,MAAMkW,eAAiB,SAASnR,EAAQwO,GAEvC,GAAIwB,GAAMnG,OAAOoG,oBAAoBzB,EAAO5E,cACxCsG,EAAQrG,OAAOoG,oBAAoBzB,EAAOsB,YAC1CsB,EAAY5C,EAAO6C,eACnBC,EAAQ9C,EAAO8B,eACf9C,EAAOwC,EAAIxT,OAAO0T,GAAO1T,OAAO4U,GAAW5U,OAAO8U,EACtD9D,GAAK+C,QAAQ/B,GACbhB,EAAK+C,QAAQvQ,GACb1E,GAAGkV,OAAOjU,MAAMV,KAAM2R,GA
EtBxN,EAAOqR,WAAavW,EAAGG,MAAMwV,YAAYT,EAAIxT,OAAO0T,GAAO1T,OAAO4U,GAAW5U,OAAOwT,GAAKxT,OAAOwD,EAAOqR,iBACvGrR,EAAOsQ,WAAaxV,EAAGG,MAAMwV,YAAYa,EAAM9U,OAAOwD,EAAOsQ,kBAOjExV,EAAGG,MAAMsW,QAAU,SAASP,GACxBA,EAAIxL,SAAOgM,YAAa,KAO5B1W,EAAGG,MAAMwW,eAAiB,SAAStP,EAAQH,GACvC,MAAQG,IAAU+K,SAASlL,EAAUuD,MAAK,UAAY,KAAO,KAOjEzK,EAAGG,MAAMyW,cAAgB,SAASxP,EAAOF,GACrC,MAAQE,IAASgL,SAASlL,EAAUuD,MAAK,SAAW,KAAO,KAO/DzK,EAAGG,MAAMyF,gBAAkB,SAASyB,EAAQH,EAAW3B,GACnD,MAAOmE,MAAKL,IAAI,EAAErJ,EAAGG,MAAMwW,eAAetP,EAAQH,GAAa3B,EAAOE,IAAMF,EAAOsR,SAMvF7W,EAAGG,MAAMwF,eAAiB,SAASyB,EAAOF,EAAW3B,GACjD,MAAOmE,MAAKL,IAAI,EAAErJ,EAAGG,MAAMyW,cAAcxP,EAAOF,GAAa3B,EAAOC,KAAOD,EAAOuR,QAMtF9W,EAAGG,MAAM4W,OAAS,SAASjU,EAAOoE,GAC9B,GAAI8P,GAAMlU,EAAM8L,UACZrJ,EAASyR,EAAIzR,SACbwR,EAASC,EAAID,SACbzS,EAAkB,MAAVyS,GAAkB,uBAA0BA,GACpD1P,EAASrH,EAAGG,MAAMyF,gBAAgB,KAAMsB,EAAW3B,GACnD6B,EAAQpH,EAAGG,MAAMwF,eAAe,KAAMuB,EAAW3B,GACjDuC,EAAIvC,EAAOC,KAAO4B,EAAM,EACxBmI,EAAIhK,EAAOE,IAAM4B,EAAO,CAG5BH,GAAUK,UAAS,KAAMc,QAEzB,IAA
I4O,GAAa/P,EAAUK,UAAS,cAAejD,KAAKA,EAExD2S,GAAWxP,QAAQC,OAAM,QACpBC,KAAI,QAAU,kBACdA,KAAI,KAAO,SACX8C,MAAK,cAAgB,UAE1BwM,EACKtP,KAAI,IAAMG,GACVH,KAAI,IAAM4H,GACV2C,KAAK,SAAS8D,GAAI,MAAOA,MAMlChW,EAAGG,MAAM+W,UAAY,SAAUhF,EAAM9K,GACjC8K,EAAK7N,KAAK,WAUN,IATA,GAEI8S,GAFAjF,EAAO1R,GAAG2G,OAAOpG,MACjBqW,EAAQlF,EAAKA,OAAOmF,MAAK,OAAQC,UAEjCtP,KACAuP,EAAa,EACbC,EAAa,IACbjI,EAAI2C,EAAKvK,KAAI,KACb8P,EAAKC,WAAWxF,EAAKvK,KAAI,OACzBgQ,EAAQzF,EAAKA,KAAK,MAAMxK,OAAM,SAAUC,KAAI,IAAM,GAAGA,KAAI,IAAM4H,GAAG5H,KAAI,KAAO8P,EAAK,MAC/EN,EAAOC,EAAM/D,OAChBrL,EAAKvE,KAAK0T,GACVQ,EAAMzF,KAAKlK,EAAK4P,KAAI,MAChBD,EAAMvN,OAAOyN,wBAA0BzQ,IACvCY,EAAKqL,MACLsE,EAAMzF,KAAKlK,EAAK4P,KAAI,MACpB5P,GAAQmP,GACRQ,EAAQzF,EAAKxK,OAAM,SAAUC,KAAI,IAAM,GAAGA,KAAI,IAAM4H,GAAG5H,KAAI,OAAS4P,EAAaC,EAAaC,EAAK,MAAMvF,KAAKiF,OAS9HnX,EAAGG,MAAM2X,YAAc,SAAUC,EAAQC,GACrC,GAAID,IAAWC,EACX,OAAO,CAEX,KAAKD,IAAWC,EACZ,OAAO,CAGX,IAAID,EAAO3V,QAAU4V,EAAO5V,OACxB,OAAO,CAEX,KAAK,GAAIY,GAAI,EACTiV,EAAIF,EAAO3V,OAAY6V,EAAJjV,EAAOA,IAE1B,GAA
I+U,EAAO/U,YAAc9B,QAAS8W,EAAOhV,YAAc9B,QAEnD,IAAKlB,EAAG8X,YAAYC,EAAO/U,GAAIgV,EAAOhV,IAClC,OAAO,MACR,IAAI+U,EAAO/U,IAAMgV,EAAOhV,GAE3B,OAAO,CAGf,QAAO,GCpsBXhD,EAAGI,OAAO8X,KAAO,WACb,YAqCA,SAASpV,GAAMsB,GAgTX,MA/SAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAI4C,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EAGjB,IAAII,GAAOJ,EAAUK,UAAS,qBAAsBjD,MAAMA,IACtDkD,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,wBAEnDwQ,GADS3Q,EAAUE,OAAM,KACrBJ,EAAKH,OAAM,KAEL,QAAViR,EACAF,EAAKE,MAAMA,IACW,OAAjBF,EAAKG,UAAsC,UAAjBH,EAAKG,WACpCH,EAAKE,MAAM1O,KAAKC,IAAIrB,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,IAAM,KAG/D0R,EAAE3F,gBAAgBC,EAAa,QAAQrR,KAAK8W,GAE5CI,EAASA,GAAUJ,EAAK5P,OAExB,IAAIiQ,GAAML,EAAKM,YACJ,OAAPD,IACAA,EAAMD,EAAOE,aAGjB,IAAIC,GAAYN,EAAE5Q,UAAS,qBACtBjD,MAAMoU,GAAiB,MAC5BD,GAAUrQ,OAAOC,SAGAxE,SAAbsO,GACAgG,EAAE5Q,UAAS,KAAMJ,OAAM,QAASsD,MAAK,YAAc0H,EAGvD,IAAIwG,GACAC,EACAC,CACJ,QAAQX,EAAKG,UACT,IAAK,MACDI,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBACjDkR,EAAI,EACyB,IAAzBvQ,EAAM7B,QAAQrE,OAChByW,EAAIC,EAA+B,EAAnBx
Q,EAAM7B,QAAQ,GAAS6B,EAAM5B,YAAc,EACzB,IAAzB4B,EAAM7B,QAAQrE,OACvByW,EAAIC,EAAYxQ,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,GAAK6B,EAAM5B,YAAc4B,EAAM7B,QAAQ,GAC9E6B,EAAM7B,QAAQrE,OAAS,IACjCyW,EAAIvQ,EAAM7B,QAAQ6B,EAAM7B,QAAQrE,OAAO,IAAIkG,EAAM7B,QAAQ,GAAG6B,EAAM7B,QAAQ,KAE1EgS,EACK9Q,KAAI,cAAgB,UACpBA,KAAI,IAAM,GACVA,KAAI,IAAMkR,EAAE,GACbE,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACTkR,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAUK,EAAMH,IAAM,QAExDhB,OAAM,QACNQ,KAAI,KAAO,UACXA,KAAI,KAAOuQ,EAAKc,eAChBrR,KAAI,cAAgB,UACpBuK,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,eACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,QAGzE,MACJ,KAAK,SACD2V,EAAeM,EAAoB,EACnC,IAAIC,GAAe,GACfC,EAAa,EACbC,EAASjB,EAAE5Q,UAAS,KAAMJ,OAAM,QAChCkS,EAAm
B,EACvB,IAAIC,EAAa,IAAK,CAElBF,EAAOzR,KAAI,YAAc,IAEzByR,EAAO/U,KAAK,SAAS8D,EAAEnF,GACnB,GAAIuW,GAAMxY,KAAKwM,wBACXnG,EAAQmS,EAAInS,KAChB+R,GAAaI,EAAIlS,OACdD,EAAQ8R,IAAcA,EAAe9R,KAE5CiS,EAAmB,UAAYC,EAAe,OAASH,EAAW,EAAIjB,EAAKc,eAAiB,GAE5F,IAAIQ,GAAM9P,KAAKC,IAAID,KAAK8P,IAAIF,EAAa5P,KAAK+P,GAAG,KACjDd,IAAgBa,EAAMA,EAAIN,EAAeA,GAAc,GAEvDE,EACKzR,KAAI,YAAc0R,GAClB5O,MAAK,cAAgB6O,EAAa,IAAM,EAAI,QAAU,WAEvDI,GACAN,EACKzR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,gBAAkBA,EAAI,GAAK,EAAI,IAAM,MAAQ,MAG5DoW,EAAOzR,KAAI,YAAc,iBAGjC8Q,GAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/CkR,EAAI,EACyB,IAAzBvQ,EAAM7B,QAAQrE,OACdyW,EAAIC,EAA+B,EAAnBxQ,EAAM7B,QAAQ,GAAS6B,EAAM5B,YAAc,EAC3B,IAAzB4B,EAAM7B,QAAQrE,OACrByW,EAAIC,EAAYxQ,EAAM7B,QAAQ,GAAK6B,EAAM7B,QAAQ,GAAK6B,EAAM5B,YAAc4B,EAAM7B,QAAQ,GAChF6B,EAAM7B,QAAQrE,OAAS,IAC/ByW,EAAIvQ,EAAM7B,QAAQ6B,EAAM7B,QAAQrE,OAAO,IAAIkG,EAAM7B,QAAQ,GAAG6B,EAAM7B,QAAQ,KAE9EgS,EACK9Q,KAAI,cAAgB,UACpBA,KAAI,IAAMgR,GACVhR,KAAI,IAAMkR,EAAE,GACbE,IAEAH,EAAatR,EAAKC,UAAS,mBAEtBjD,MAAMgE,E
AAM3B,SAAS,GAAI2B,EAAM3B,SAAS2B,EAAM3B,SAASvE,OAAS,KACrEwW,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACTkR,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAWK,EAAMH,IAAM2Q,EAAYxQ,EAAM5B,YAAc,EAAI,IAAO,QAEpGS,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAMuQ,EAAKc,eACfrR,KAAI,YAAc0R,GAClB5O,MAAK,cAAgB6O,EAAgBA,EAAa,IAAM,EAAI,QAAU,MAAS,UAC/EpH,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,kBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,aAAehD,EAAGG,MAAM8H,UAAWK,EAAMH,IAAM2Q,EAAYxQ,EAAM5B,YAAc,EAAI,IAAO,QAI7G,MACJ,KAAK,QACD+R,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/C8Q,EACKhO,MAAK,cAAgBkP,EAAe,SAAW,SAC/ChS,KAAI,YAAcgS,EAAe,aAAe,IAChDhS,KAAI,IAAMgS,GAAiBjQ,KAAKL,IAAI9D,EAAOuR,MAAO1P,GAAS,IAAM6R,GAAqB,GAAM,KAC5FtR,KAAI,IAAMgS,EAAgBnZ,GAAG6I,IAAIf,EAAM7B,SAAW,EAAKyR,EAAKc,eAC7DD,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS
,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KAAI,OAC7FlQ,OAAM,QACJ+C,MAAK,UAAY,GACtBmO,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAMH,IAAM,MAE1DhB,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAM,GACVA,KAAI,IAAMuQ,EAAKc,eACfvO,MAAK,cAAgB,SACrByH,KAAK,SAAS/J,EAAGnF,GACd,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,iBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,MAElEmE,OAAM,QACNsD,MAAK,UAAY,GAE1B,MACJ,KAAK,OASDgO,EAAUhR,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC/C8Q,EACKhO,MAAK,cAAgBkP,EAAe,SAAW,OAC/ChS,KAAI,YAAcgS,EAAe,cAAgB,IACjDhS,KAAI,IAAMgS,GAAiBjQ,KAAKL,IAAI9D,EAAOC,KAAM4B,GAAS,IAAM6R,GAAqB,GAAM,KAC3FtR,KAAI,IAAMgS,GAAiBnZ,GAAG6I,IAAIf,EAAM7B,SAAW,GAAMyR,EAAKc,eAC/DD,IACAH,EAAatR,EAAKC,UAAS,mBACtBjD,KAAKgE,EAAM3B,UAChBiS,EAAWnR,QAAQC,OAAM,KAAMC,KAAI,QAAS,SAASQ,EAAEnF,GAC/C,OAAO,gBAAe,kBAA0B,GAALA,EAAS,eAAa,gBAAmB4U,KA
AI,OAC7FlQ,OAAM,QACJ+C,MAAK,UAAY,GACtBmO,EAAWxQ,OAAOC,SAClBuQ,EACKjR,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUqQ,EAAOnQ,IAAM,MAE3DhB,OAAM,QACNQ,KAAI,KAAO,SACXA,KAAI,IAAM,GACVA,KAAI,KAAOuQ,EAAKc,eAChBrR,KAAI,cAAgB,OACpBuK,KAAK,SAAS/J,EAAEnF,GACb,GAAIkG,GAAIqP,EAAIpQ,EACZ,QAAO,GAAMe,GAAG5D,MAAK,OAAU,GAAK4D,IAE5C0P,EAAWpG,gBAAgBC,EAAa,iBACnC9K,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,eAAiBhD,EAAGG,MAAM8H,UAAUK,EAAM7B,QAAQzD,IAAM,MAElEmE,OAAM,QACNsD,MAAK,UAAY,IA2BlC,GAvBAgO,EAAUvG,KAAK,SAAS/J,GAAK,MAAOA,MAEhC4Q,GAAiC,SAAlBb,EAAKG,UAAyC,UAAlBH,EAAKG,WAEhDF,EAAE5Q,UAAS,KACNlD,KAAK,SAAS8D,EAAEnF,GACbxC,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASQ,KAAI,UAAY,IAC3CW,EAAMH,GAAKG,EAAM7B,QAAQ,GAAK,IAAM6B,EAAMH,GAAKG,EAAM7B,QAAQ,GAAK,OAC9D0B,EAAI,OAAa,OAAJA,IACb3H,GAAG2G,OAAOpG,MAAM4G,KAAI,UAAY,GAEpCnH,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASQ,KAAI,UAAY,MAKvDW,EAAM3B,SAAS,IAAM2B,EAAM3B,SAAS,IAA2B,GAArB2B,EAAM3B,SAAS,IACzDW,EAAKC,UAAS,mBAAoBkD,MAAK,UAAY,SAAUtC,EAAGnF,GAC5D,MAAQA,GAAQ,EAAJ,KAKpB+V,IAAiC,QAAlBb
,EAAKG,UAAwC,WAAlBH,EAAKG,UAAwB,CACvE,GAAIuB,KACJtS,GAAKC,UAAS,mBACTlD,KAAK,SAAS8D,EAAEnF,GACb,IACQA,EACA4W,EAAYnW,KAAK6E,EAAMH,GAAKpH,KAAKwM,wBAAwBnG,MAAQ,GAEjEwS,EAAYnW,KAAK6E,EAAMH,GAAKpH,KAAKwM,wBAAwBnG,MAAQ,GACxE,MAAOyS,GACA7W,EACA4W,EAAYnW,KAAK6E,EAAMH,GAAK,GAE5ByR,EAAYnW,KAAK6E,EAAMH,GAAK,MAI5CgQ,EAAE5Q,UAAS,KAAMlD,KAAK,SAAS8D,EAAGnF,IAC1BsF,EAAMH,GAAKyR,EAAY,IAAMtR,EAAMH,GAAKyR,EAAY,MAChDzR,EAAI,OAAa,OAAJA,EACb3H,GAAG2G,OAAOpG,MAAMsH,SAEhB7H,GAAG2G,OAAOpG,MAAMoG,OAAM,QAASkB,YAM/C8P,EAAE5Q,UAAS,SACNkF,OAAO,SAAUtE,GAMd,OAAQuP,WAAWhO,KAAK6E,MAAU,IAAJpG,GAAc,MAAmBtE,SAANsE,IAE5DuC,QAAO,QAAS,GAGrB4N,EAAShQ,EAAMwR,SAInBrH,EAAYS,UAAS,kBACdpQ,EA/UX,GAAIoV,GAAO1X,GAAG0V,IAAIgC,OACd5P,EAAQ9H,GAAG8H,MAAMC,SAEjBhD,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C4B,EAAQ,GACRC,EAAS,GACTqR,EAAgB,KAChBK,GAAa,EACbO,EAAe,EACfK,GAAe,EACfD,GAAgB,EAChBZ,GAAY,EACZV,EAAQ,KACRa,EAAoB,EACpB9G,EAAWtO,OACX4E,EAAW,IACX/H,EAAWF,GAAGE,SAAQ,YAE5BwX,GACK5P,MAAMA,GACN+P,OAAM,UACNG,WAAW,SAASrQ,GAAK,MAAOA,
IAOrC,IAAImQ,GACA7F,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EAkWjD,OAxCA3F,GAAMoV,KAAOA,EACbpV,EAAMpC,SAAWA,EAEjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAC1CA,EAAMgM,SAAWC,OAAOC,WAEpBiK,mBAAoBhK,IAAK,WAAW,MAAOgK,IAAqB/J,IAAK,SAASvG,GAAGsQ,EAAkBtQ,IACnG+Q,eAAoBzK,IAAK,WAAW,MAAOyK,IAAiBxK,IAAK,SAASvG,GAAG+Q,EAAc/Q,IAC3F2Q,cAAoBrK,IAAK,WAAW,MAAOqK,IAAgBpK,IAAK,SAASvG,GAAG2Q,EAAa3Q,IACzFgR,cAAoB1K,IAAK,WAAW,MAAO0K,IAAgBzK,IAAK,SAASvG,GAAGgR,EAAahR,IACzFoQ,YAAoB9J,IAAK,WAAW,MAAO8J,IAAc7J,IAAK,SAASvG,GAAGoQ,EAAWpQ,IACrF8P,WAAoBxJ,IAAK,WAAW,MAAOyJ,IAAiBxJ,IAAK,SAASvG,GAAG+P,EAAc/P,IAC3FtB,QAAoB4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IAC7EyP,OAAoBnJ,IAAK,WAAW,MAAOmJ,IAASlJ,IAAK,SAASvG,GAAGyP,EAAMzP,IAC3EvB,OAAoB6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAC3EwJ,UAAoBlD,IAAK,WAAW,MAAOkD,IAAYjD,IAAK,SAASvG,GAAGwJ,EAASxJ,IAGjFpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAmB5B,SAAV8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAqBjT,SAAZ8E,EAAEmO,MAA
uBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAoB3B,SAAX8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/DiD,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAASE,EACT8J,EAAYW,MAAM3K,KAEtBH,OAAQ2G,IAAK,WAAW,MAAO3G,IAAS4G,IAAK,SAASvG,GAClDL,EAAQK,EACRuP,EAAK5P,MAAMA,GACXwQ,EAAwC,kBAArBxQ,GAAMjC,WACzBrG,EAAGG,MAAMkV,iBAAiBvS,EAAOwF,GAAO,SAAW,QAAS,YAAa,mBAIjFtI,EAAGG,MAAMqP,YAAY1M,GACrB9C,EAAGG,MAAMkV,iBAAiBvS,EAAOoV,GAAM,SAAW,aAAc,gBAAiB,WAAY,cAAe,eAC5GlY,EAAGG,MAAMkV,iBAAiBvS,EAAOwF,GAAO,SAAW,QAAS,YAAa,eAElExF,GCtYX9C,EAAGI,OAAO2Z,QAAU,WAChB,YAsCA,SAASjX,GAAMsB,GA+NX,MA9NAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAIqB,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,GAGjBd,EAAOO,OAAOqT,GAAW1V,EAAKmQ,IAAI,SAAStM,EAAEnF,GAAK,MAAOiX,GAAK9R,EAAEnF,MAC3DqD,WAAW6T,IAAW,EAAGvU,GAAiB,GAG/C,IAAIwU,KACJ,KAAKC,EAAS,CAEV,GAAiBC,GAAMC,EAAnBzR,IACJvE,GAAK2F,QAAQ
,SAAU9B,EAAGnF,GACtB,GAAIuX,GAAKC,EAAMrS,GAAIsS,EAAKC,EAAMvS,GAAIwS,EAAKC,EAAMzS,GAAI0S,EAAKC,EAAM3S,GACxD4S,EAAUC,EAAW7S,EACrB4S,IACAA,EAAQ9Q,QAAQ,SAAUrI,EAAGoB,GACzB6F,EAAOpF,KAAKwX,EAAWrZ,EAAGoB,EAAGa,WAGjC8W,GAAM9R,EAAOpF,KAAKkX,GAClBJ,GAAM1R,EAAOpF,KAAK8W,GAClBE,GAAM5R,EAAOpF,KAAKgX,GAClBI,GAAMhS,EAAOpF,KAAKoX,KAE1BR,EAAO7Z,GAAGgJ,IAAIX,GACdyR,EAAO9Z,GAAG6I,IAAIR,GACdsR,GAAUE,EAAMC,GAGpBY,EAAOvU,OAAOyT,GAAWD,GACzBe,EAAOzU,MAAM0U,IAAWvV,EAAiB,IAGzCwV,EAAUA,GAAWhV,EACrBiV,EAAUA,GAAWH,EAAOpB,OAAOrT,OAAOyU,EAAO,GAAGA,EAAO,IAG3D,IAAI5T,GAAOJ,EAAUK,UAAS,aAAcjD,MAAMA,GAClCgD,GAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,eACvDL,GAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAEvE,IAAI6V,GAAWhU,EAAKC,UAAS,eAAgBjD,KAAK,SAAS6D,GAAK,MAAOA,KACnEoT,EAAWD,EAAS7T,QAAQC,OAAM,KAAM+C,MAAK,iBAAmB,MAAMA,MAAK,eAAiB,KAChG6Q,GACK3T,KAAI,QAAU,cACdA,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAAK,MAAO,cAAgBpV,EAAO6T,EAAK9R,EAAEnF,IAA2B,IAArBoD,EAAOM,aAAsB,SAC5GgE,QAAO,QAAU,SAASvC,GAAK,MAAOA,GAAEsT,QAC7CH,EACK9I,gBAAgBC,EAAa,
wBAC7BhI,MAAK,iBAAmB,GACxBA,MAAK,eAAiB,KACtB2D,MAAM,SAASjG,EAAEnF,GAAK,MAAOA,GAAIyF,EAAWnE,EAAKlC,SACjDuF,KAAI,YAAc,SAASQ,EAAEnF,GAC1B,MAAO,cAAgBoD,EAAO6T,EAAK9R,EAAEnF,IAA2B,IAArBoD,EAAOM,aAAsB,SAEhF4U,EAASlT,OAAOC,SAKhBkT,EAASlX,KAAK,SAAS8D,EAAEnF,GACrB,GAAIuW,GAAM/Y,GAAG2G,OAAOpG,OACnB6Z,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAa7X,SAAT6X,EAAEvT,IAA6B,OAATuT,EAAEvT,GAAa,CACrC,GAAIqE,GAAOkP,IAAMd,EAAS,MAAQ,MAClCrB,GAAI7R,OAAM,QACP+C,MAAK,SAAWgG,EAAStI,IAAMmE,EAAMnE,EAAEnF,IACvC2E,KAAI,QAAU,iCAAmC6E,GACpD+M,EAAI7R,OAAM,QACP+C,MAAK,SAAWgG,EAAStI,IAAMmE,EAAMnE,EAAEnF,IACvC2E,KAAI,QAAU,8BAAgC6E,OAK7D,IAAImP,GAAY,WAAa,MAAwB,QAAhBC,EAA4C,GAArBxV,EAAOM,YAAoBgD,KAAKF,IAAI,GAAyB,GAArBpD,EAAOM,cACvGmV,EAAY,WAAa,MAA4B,IAArBzV,EAAOM,YAAqBiV,IAAY,GACxEG,EAAY,WAAa,MAA4B,IAArB1V,EAAOM,YAAqBiV,IAAY,IAG3Ef,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAIlP,GAAOkP,IAAMd,EAAS,MAAQ,OAC9BmB,EAAYL,IAAMd,EAASJ,EAAQE,CACvCY,GAASnU,OAAM,sCAAyCqF,GACrDgG,gBAAgBC,EAAa,wBAC3B9K,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,E
AAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,MAC3CR,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOa,EAAS5T,MACvDmT,EAASnU,OAAM,mCAAsCqF,GAClDgG,gBAAgBC,EAAa,wBAC3B9K,KAAI,KAAOkU,GACXlU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,MAC3CR,KAAI,KAAOmU,GACXnU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOQ,EAAEvT,SAGnDyS,EAAOE,GAAO7Q,QAAQ,SAAUyR,GAC7B,GAAIlP,GAAOkP,IAAMd,EAAS,MAAQ,MAClCW,GAAShU,UAAS,eAAkBiF,GACjC7K,GAAE,YAAc,SAASwG,EAAEnF,EAAEwY,GAC1Bhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACL5P,QAAUI,IAAKkP,EAAEvT,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEqT,IACnD5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,EAAEwY,GACzBhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLoG,QAAUI,IAAKkP,EAAEvT,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEqT,IACnD5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,YAK3CwW,EAAS7T,OAAM,QACVC,KAAI,QAAU,kBAEdhG,GAAE,YAAc,SAASwG,EAAEnF,GACxBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACLxP,IAAKyN,EAAK9R,
GACV6D,MAAOiO,EAAK9R,GACZiE,SACMI,IAAK,KAAMR,MAAO0O,EAAMvS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOiQ,EAAM9T,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOwO,EAAMrS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAEhEsB,KAAM6D,EACNiB,MAAOpG,EACPpB,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,GACvBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLwG,IAAKyN,EAAK9R,GACV6D,MAAOiO,EAAK9R,GACZiE,SACMI,IAAK,KAAMR,MAAO0O,EAAMvS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOiQ,EAAM9T,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAC1DwJ,IAAK,KAAMR,MAAOwO,EAAMrS,GAAImE,MAAOmE,EAAStI,IAAMmE,EAAMnE,EAAEnF,KAEhEsB,KAAM6D,EACNiB,MAAOpG,EACPpB,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,UAIzCuW,EAASnU,OAAM,uBACZqL,gBAAgBC,EAAa,qBAC3B9K,KAAI,IAAM,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOR,EAAMvS,MAC9CR,KAAI,QAAUgU,GACdhU,KAAI,IAAMkU,GACVlU,KAAI,SAAW,SAASQ,EAAEnF,GAAK,MAAO0G,MAAKC,IAAIuR,EAAOR,EAAMvS,IAAM+S,EAAOV,EA
AMrS,MAAQ,IACvFsC,MAAK,OAAS,SAAStC,EAAEnF,GAAK,MAAOyN,GAAStI,IAAMmE,EAAMnE,EAAEnF,KAC5DyH,MAAK,SAAW,SAAStC,EAAEnF,GAAK,MAAOyN,GAAStI,IAAMmE,EAAMnE,EAAEnF,KAGnEuY,EAAS7T,OAAM,QAASC,KAAI,QAAU,qBAEtC2T,EAASnU,OAAM,0BACZqL,gBAAgBC,EAAa,6BAC3B9K,KAAI,KAAOkU,GACXlU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOe,EAAM9T,MAC/CR,KAAI,KAAOmU,GACXnU,KAAI,KAAO,SAASQ,EAAEnF,GAAK,MAAOkY,GAAOe,EAAM9T,KAGpD,IAAI+T,GAAWZ,EAAS/T,UAAS,uBAAwBjD,KAAK,SAAS6D,GACnE,MAAO6S,GAAW7S,QAEtB+T,GAASzU,QAAQC,OAAM,UAClB+C,MAAK,OAAS,SAAStC,EAAEnF,EAAEwY,GAAK,MAAOW,GAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,KACpE/Q,MAAK,SAAW,SAAStC,EAAEnF,EAAEwY,GAAK,MAAOW,GAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,KACtE/Q,MAAK,UAAY,KACjB9I,GAAE,YAAc,SAASwG,EAAEnF,EAAEwY,GAC1Bhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsb,kBACL5P,QAAUI,IAAK4P,EAAWjU,EAAEnF,EAAEwY,GAAIlP,MAAO6P,EAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,IACtE5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,WAAa,SAASwG,EAAEnF,EAAEwY,GACzBhb,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACLoG,QA
AUI,IAAK4P,EAAWjU,EAAEnF,EAAEwY,GAAIlP,MAAO6P,EAAWhU,EAAEnF,EAAEwY,IAAMlP,EAAMnE,EAAEqT,IACtE5Z,EAAGpB,GAAGuE,UAGbpD,GAAE,YAAc,SAASwG,EAAEnF,GACxBtC,EAASmG,kBAAkBjF,EAAGpB,GAAGuE,UAEzCmX,EAASvU,KAAI,QAAU,sBACvBuU,EACG1J,gBAAgBC,EAAa,kCAC3B9K,KAAI,KAA4B,IAArBvB,EAAOM,aAClBiB,KAAI,KAAO,SAASQ,EAAEnF,EAAEwY,GAAK,MAAON,GAAOD,EAAW9S,EAAEnF,EAAEwY,MAC1D7T,KAAI,IAAM,KACfuU,EAAS9T,OAAOC,SAGhB+S,EAAUhV,EAAO0T,OACjBuB,EAAUH,EAAOpB,SAGrBrH,EAAYS,UAAS,wBACdpQ,EA/PX,GAmBIkX,GAASE,EACTE,EAASe,EASTC,EAASC,EA7BT9V,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC7C4B,EAAQ,IACRC,EAAS,IACTkD,EAAKb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UACrB/E,EAAS5F,GAAG8H,MAAMsI,UAClBsK,EAAS1a,GAAG8H,MAAMC,SAClB0R,EAAQ,SAAS9R,GAAK,MAAOA,GAAEkU,OAC/B7B,EAAQ,SAASrS,GAAK,MAAOA,GAAEU,OAAOyT,IACtCL,EAAQ,SAAS9T,GAAK,MAAOA,GAAEU,OAAO0T,IACtC7B,EAAQ,SAASvS,GAAK,MAAOA,GAAEU,OAAO2T,IACtC5B,EAAQ,SAASzS,GAAK,MAAOA,GAAEU,OAAO4T,aACtC3B,EAAQ,SAAS3S,GAAK,MAAOA,GAAEU,OAAO6T,cACtCjM,EAAW,SAAStI,GAAK,MAAOA,GAAEmE,OAClC0O,EAAc,SAAS7S,GAAK,MAAOA,
GAAEU,OAAOqT,UAC5CjB,EAAa,SAAS9S,EAAGnF,EAAGwY,GAAK,MAAOrT,IACxCiU,EAAa,SAASjU,EAAGnF,EAAGwY,GAAK,MAAOrT,IACxCgU,EAAa,SAAShU,EAAGnF,EAAGwY,GAAK,MAAO3X,SACxCyI,EAAQtM,EAAGG,MAAMuQ,eACjBxJ,EAAY,KAGZxG,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,mBAAoB,aAClF+H,EAAW,IACXmT,EAAc,KAOdnJ,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EA8RjD,OAtDA3F,GAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAc6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACrEtB,QAAc4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACvEiT,aAAc3M,IAAK,WAAW,MAAO2M,IAAe1M,IAAK,SAASvG,GAAGiT,EAAYjT,IACjFb,GAAcmH,IAAK,WAAW,MAAOgL,IAAQ/K,IAAK,SAASvG,GAAGsR,EAAKtR,IACnE4R,IAAKtL,IAAK,WAAW,MAAOuL,IAAStL,IAAK,SAASvG,GAAG6R,EAAM7R,IAC5DgU,IAAK1N,IAAK,WAAW,MAAOgN,IAAS/M,IAAK,SAASvG,GAAGsT,EAAMtT,IAC5D8R,IAAKxL,IAAK,WAAW,MAAOyL,IAASxL,IAAK,SAASvG,GAAG+R,EAAM/R,IAC5DgS,IAAK1L,IAAK,WAAW,MAAO2L,IAAS1L,IAAK,SAASvG,GAAGiS,EAAMjS,IAC5DkS,IAAK5L,IAAK,WAAW,MAAO6L,IAAS5L,IAAK,SAASvG,GAAGmS,EAAMnS,IAC5DiU
,WAAe3N,IAAK,WAAW,MAAOwB,IAAYvB,IAAK,SAASvG,GAAG8H,EAAS9H,IAC5EuT,UAAejN,IAAK,WAAW,MAAO+L,IAAc9L,IAAK,SAASvG;AN3R1E,AM2R6EqS,EN3R3E,AM2RsFrS,CN3RrF,GAAG,AM4REkU,CN5RD,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EM4RK5N,IN5RC,AM4RI,WAAW,MAAOgM,IAAc/L,IAAK,SAASvG,GAAGsS,EAAWtS,IAChFmU,cAAe7N,IAAK,WAAW,MAAOmN,IAAclN,IAAK,SAASvG,GAAGyT,EAAWzT,IAChFoU,cAAe9N,IAAK,WAAW,MAAOkN,IAAcjN,IAAK,SAASvG,GAAGwT,EAAWxT,IAChFvC,QAAU6I,IAAK,WAAW,MAAO7I,IAAU8I,IAAK,SAASvG,GAAGvC,EAAOuC,IACnEuS,QAAUjM,IAAK,WAAW,MAAOiM,IAAUhM,IAAK,SAASvG,GAAGuS,EAAOvS,IACnEqR,SAAU/K,IAAK,WAAW,MAAO+K,IAAW9K,IAAK,SAASvG,GAAGqR,EAAQrR,IACrEyR,SAAUnL,IAAK,WAAW,MAAOmL,IAAWlL,IAAK,SAASvG,GAAGyR,EAAQzR,IACrEuR,QAAUjL,IAAK,WAAW,MAAOiL,IAAUhL,IAAK,SAASvG,GAAGuR,EAAOvR,IACnEwS,QAAUlM,IAAK,WAAW,MAAOkM,IAAUjM,IAAK,SAASvG,GAAGwS,EAAOxS,IACnE4B,IAAc0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,GAAG4B,EAAG5B,IAE/D4G,GACIN,IAAK,WAED,MADA9M,SAAQK,KAAI,mFAGhB0M,IAAK,SAASvG,GACVxG,QAAQK,KAAI,iFAIpB+C,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAA
sB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,KAE9BF,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAAWE,EACX8J,EAAYW,MAAM3K,OAI1BzI,EAAGG,MAAMqP,YAAY1M,GAEdA,GCnUX9C,EAAGI,OAAO4c,aAAe,WACrB,YA0CA,SAASla,GAAMsB,GAuHX,MAtHAqO,GAAYW,QACZX,EAAYrS,OAAO6c,GACfC,GAAWzK,EAAYrS,OAAO+c,GAC9BC,GAAW3K,EAAYrS,OAAOid,GAElCjZ,EAAUC,KAAK,SAASC,GACpB,GAAI4C,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EACjB,IAAIvB,IAAkByB,GAAUgL,SAASlL,EAAUuD,MAAK,WAAc,KAAOlF,EAAOC,KAAOD,EAAOuR,MAC9FlR,GAAmByB,GAAU+K,SAASlL,EAAUuD,MAAK,YAAe,KAAOlF,EAAOE,IAAMF,EAAOsR,MAUnG,IARA/T,EAAMqR,OAAS,WACXzT,EAAS4c,eACTpW,EAAUiH,aAAa1F,SAASA,GAAUrH,KAAK0B,IAEnDA,EAAMoE,UAAYnG,MAIbuD,IAASA,EAAKlC,OAAQ,CACvB,GAAI6U,GAAa/P,EAAUK,UAAS,cAAejD,MAAMyS
,GAYzD,OAVAE,GAAWxP,QAAQC,OAAM,QACpBC,KAAI,QAAU,kBACdA,KAAI,KAAO,SACX8C,MAAK,cAAgB,UAE1BwM,EACKtP,KAAI,IAAMpC,EAAOC,KAAOG,EAAiB,GACzCgC,KAAI,IAAMpC,EAAOE,IAAMG,EAAkB,GACzCsM,KAAK,SAAS/J,GAAK,MAAOA,KAExBrF,EAEPoE,EAAUK,UAAS,cAAec,SAItCP,EAAImV,EAAQ7W,SACZmJ,EAAI0N,EAAQ/B,SAASqC,OAAM,EAG3B,IAAIjW,GAAOJ,EAAUK,UAAS,gCAAiCjD,MAAMA,IACjEkZ,EAASlW,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,mCAAmCD,OAAM,KACzF+V,EAAYD,EAAO9V,OAAM,QACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,gBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAC5BD,OAAM,KAAMC,KAAI,QAAU,eAC1BD,OAAM,QAEX8V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,eACjCwQ,EAAExQ,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,KAEhEiY,GACAvF,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,aAAehC,EAAiB,OAI3DsX,EAAQ7V,MAAMzB,GAAgB0B,OAAOzB,EAErC,IAAI+X,GAAWxF,EAAEhR,OAAM,gBAClByW,MAAMtZ,EAAKmI,OAAO,SAAStE,GAAK,OAAQA,EAAE0V,WAc/C,IAZAF,EAASxP,aAAa/M,KAAK6b,GAE3BQ,EAAU/V,OAAM,YACXC,KAAI,KAAO,mBAAqBsV,EAAQ1S,MACxC7C,OAAM,QAEXyQ,EAAEhR,OAAM,oBAAuB8V,EAAQ1S,KAAO,SACzC5C,KAAI,QAAUG,EAAEpB,aAAeg
T,EAAgB,EAAI,IACnD/R,KAAI,SAAW,IACfA,KAAI,KAAOG,EAAEpB,aAAegT,EAAgB,EAAI,IAGjDwD,EAAW,CACXC,EACK7U,MAAMR,GACNsQ,MAAOpY,EAAGG,MAAMuU,WAAW/O,EAAe,IAAKrB,IAC/CwZ,UAAUlY,EAAiB,GAEhCuS,EAAEhR,OAAM,iBAAkBQ,KAAI,YAAc,eAAiB4H,EAAE9I,QAAQ,GAAK,KAC5E0R,EAAEhR,OAAM,iBAAkB/F,KAAK+b,EAE/B,IAAI/D,GAASjB,EAAEhR,OAAM,iBAAkBI,UAAS,IAC5CmS,IACAN,EACK7R,UAAS,QACTI,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAAK,MAAO,gBAAkBA,EAAI,IAAM,EAAI,IAAM,MAAQ,MAIlG4B,IACAC,EACK/U,MAAMiH,GACN6I,MAAO1O,KAAKwB,MAAMtF,EAAgB,KAClCkY,UAAWnY,EAAgB,GAEhCwS,EAAEhR,OAAM,iBAAkB/F,KAAKic,IAInClF,EAAEhR,OAAM,qBACHQ,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAO4H,EAAE,IACb5H,KAAI,KAAO4H,EAAE,MAQtBkD,EAAYS,UAAS,8BACdpQ,EA3JX,GAaIgF,GAAGyH,EAbH0N,EAAUjd,EAAGI,OAAO2Z,UACpBoD,EAAQnd,EAAGI,OAAO8X,OAClBmF,EAAQrd,EAAGI,OAAO8X,OAElB3S,GAAUE,IAAK,GAAIqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,IAChD4B,EAAQ,KACRC,EAAS,KACTiF,EAAQtM,EAAGG,MAAMsQ,WACjByM,GAAY,EACZE,GAAY,EACZM,GAAkB,EAClBhE,GAAgB,EAChBxZ,EAAUF,EAAGI,OAAOF,UAEpB6W,EAAS,qBACTrW,EAAWF,GAAGE,SAAQ,eAAiB
,aACvC+H,EAAW,GAEf0U,GACK9E,OAAM,UACNU,YAAW,GACXP,WAAW,SAASrQ,GAAK,MAAOA,KAErCkV,EACKhF,OAAM,EAAqB,QAAU,QACrCG,WAAWhY,GAAGmM,OAAM,SAGzBzM,EAAQuI,SAAS,EAMjB,IAAIgK,GAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,EA6LjD,OA7DAwU,GAAQvc,SAASiB,GAAE,2BAA6B,SAASoc,GACrD7d,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+W,EAAQvc,SAASiB,GAAE,0BAA4B,SAASoc,GACpD7d,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+W,EAAQvc,SAASiB,GAAE,2BAA6B,SAASoc,GACrD7d,MAOJ4C,EAAMpC,SAAWA,EACjBoC,EAAMma,QAAUA,EAChBna,EAAMqa,MAAQA,EACdra,EAAMua,MAAQA,EACdva,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAa6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACpEtB,QAAa4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACtE+Q,eAAgBzK,IAAK,WAAW,MAAOyK,IAAiBxK,IAAK,SAASvG,GAAG+Q,EAAc/Q,IACvFuU,WAAYjO,IAAK,WAAW,MAAOiO,IAAahO,IAAK,SAASvG,GAAGuU,EAAUvU,IAC3EyU,WAAYnO,IAAK,WAAW,MAAOmO,IAAalO,IAAK,SAASvG,GAAGyU,EAAUzU,IAC3EqV,gBAAoB/O,IAAK,WAAW,MAAO/O,IAAWgP,IAAK,SAASvG,GAAGzI,EAAQyI,IAC/EoO,QAAY9H,IAAK,WAA
W,MAAO8H,IAAU7H,IAAK,SAASvG,GAAGoO,EAAOpO,IAGrEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/DiD,UAAWwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GACxDF,EAAWE,EACX8J,EAAYW,MAAM3K,GAClBwU,EAAQxU,SAASA,GACjB0U,EAAM1U,SAASA,GACf4U,EAAM5U,SAASA,KAEnB6D,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,GAC1BsU,EAAQ3Q,MAAMA,KAElBoR,iBAAkBzO,IAAK,WAAW,MAAOyO,IAAmBxO,IAAK,SAASvG,GACtE+U,EAAkB/U,EAClB0U,EAAMhF,OAAQ,EAAM,QAAU,YAItCrY,EAAGG,MAAMkW,eAAevT,EAAOma,GAC/Bjd,EAAGG,MAAMqP,YAAY1M,GAEdA,GCjOX9C,EAAGI,OAAO6d,OAAS,WACf,YA6BA,SAASC,GAAWC,EAAQtV,GACxB,GAAIuV,GAAKD,EAAOhd,OAChBgd,GAAOvI,KAAK,SAAS9F,EAAGuO,GACpB,GAAIC,GAAKF,EAAGjL,QAAQrD,GAChByO,EAAKH,EAAGjL,QAAQkL,EACpB,OAAO7d,IAAGge,WAAW3V,EAAOyV,GAAKzV,EAAO0V,M
AIhD,QAASzb,GAAMsB,GAkOX,MAjOAA,GAAUC,KAAK,SAAS8D,EAAGnF,GACvB,GAAI2C,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,EAEjB,IAAIuX,GAASC,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACjCwd,EAAUC,EAAQxd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACnC0d,EAAcC,EAAY1d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC3C4d,EAAWC,EAAS5d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACrC8d,EAAcC,EAAY9d,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC3Cge,EAAeC,EAAahe,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAC7Cke,EAAmBC,EAAiBle,KAAKL,KAAMoH,EAAGnF,GAAG7B,QACrDoe,EAAgBC,EAAcpe,KAAKL,KAAMoH,EAAGnF,GAAG7B,OAGnD+c,GAAWe,EAAaR,GACxBP,EAAWiB,EAAcR,GACzBT,EAAWmB,EAAkBR,GAC7BX,EAAWqB,EAAeR,GAG1BN,EAAO7I,KAAKpV,GAAGge,YACfG,EAAQ/I,KAAKpV,GAAGge,YAChBK,EAAYjJ,KAAKpV,GAAGge,YACpBO,EAASnJ,KAAKpV,GAAGge,WAIjB,IAAIiB,GAAKjf,GAAG8H,MAAMC,SACb5B,OAAQnG,GAAGkf,OAAOlf,GAAGmf,OAAOC,EAAQnB,MACpChY,MAAM6Q,GAAW3R,EAAgB,IAAM,EAAGA,GAGtC5E,MAAK8e,WAAarf,GAAG8H,MAAMC,SAC/B5B,QAAQ,EAAGoD,EAAAA,IACXtD,MAAMgZ,EAAGhZ,QAGd1F,MA
AK8e,UAAYJ,CAYjB,KAAI,GALAnY,IALW9G,GAAGgJ,IAAIiV,GACPje,GAAG6I,IAAIoV,GACPA,EAAO,GAGXvX,EAAUK,UAAS,uBAAwBjD,MAAM6D,KACxDX,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,0BACnD6V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,KAEXnE,EAAE,EAAE8c,EAAGrB,EAAOrc,OAAU0d,EAAF9c,EAAMA,IAAC,CACjC,GAAI+c,GAAkB,oBAAoB/c,CAClC,IAALA,IACC+c,EAAkBA,EAAkB,YAAYC,EAAsBhd,IAE1Ewa,EAAO9V,OAAM,QAASC,KAAI,QAAUoY,GAGxCvC,EAAO9V,OAAM,QAASC,KAAI,QAAU,cAEpCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAOvE,KAAI,GAJAwa,GAAK,SAAS9X,GAAK,MAAOuB,MAAKC,IAAI8V,EAAGtX,GAAKsX,EAAG,KAE9CS,EAAM,SAAS/X,GAAK,MAAesX,GAAJ,EAAJtX,EAAWA,EAAQ,IAE1CnF,EAAE,EAAE8c,EAAGrB,EAAOrc,OAAU0d,EAAF9c,EAAMA,IAAC,CACjC,GAAIyD,GAAQgY,EAAOzb,EACnBmV,GAAEhR,OAAM,gBAAiBnE,GACpB4a,MAAMnX,GACNkB,KAAI,SAAW/B,GACfuI,aACA1F,SAASA,GACTd,KAAI,QAAUsY,EAAGxZ,IACjBkB,KAAI,IAAMuY,EAAIzZ,IAGvB0R,EAAEhR,OAAM,mBACHsD,MAAK,OAAS6B,GACd3E,KAAI,SAAW/B,EAAkB,GACjC+B,KAAI,IAAM/B,EAAkB,GAC5BjE,GAAE,YAAc,WACbjB,EAASsb,kBACLhQ,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,U
AC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,YAAc,WACbjB,EAASmG,kBACLmF,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,UAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,WACZjB,EAASsF,iBACLgG,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,UAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC0D,aACA1F,SAASA,GACTd,KAAI,QAAqB,EAAXoX,EACXU,EAAG,GAAKA,EAAGV,EAAS,IAClBU,EAAGV,EAAS,IAAMU,EAAG,IAC1B9X,KAAI,IAAMuY,EAAInB,GAEnB,IAAIoB,GAAMva,EAAkB,EAExBwa,EAAazB,EAAQlK,IAAK,SAAS4L,EAAQjX,GAC3C,OAAQ4C,MAAOqU,EAAQhE,MAAO8C,EAAa/V,KAE/CoU,GACGjW,UAAS,0BACTjD,KAAK8b,GACL3Y,QACAC,OAAM,QACNC,KAAI,QAAU,qBACdA,KAAI,IAAM,MAAQwY,EAAK,IAAMA,EAAK,KAAQA,EAAM,KAAQA,EAAM,KAAQA,EAAM,KAC5Exe,GAAE,YAAc,SAASwG,GACxBzH,EAASsb,kBACPhQ,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,QAC5BwC,KAAMwS,EAAGtX,EAAE6D,OAAQpG,EAAgB,OAItCjE,GAAE,YAAc,SAASwG,GACtBzH,EAASmG,kBACLmF,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAGnF,GACxBtC,EA
ASsF,iBACLgG,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAItC0N,EAAE5Q,UAAS,0BACRjD,KAAK8b,GACLjS,aACA1F,SAASA,GACTd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,EAAE6D,OAAS,IAAOpG,EAAkB,EAAK,KAErG,IAAI0a,GAAkBzB,EAAYpK,IAAK,SAAS4L,EAAQjX,GACpD,OAAQ4C,MAAOqU,EAAQhE,MAAOgD,EAAiBjW,KAEnDoU,GACGjW,UAAS,sBACTjD,KAAKgc,GACL7Y,QACAC,OAAM,QACNC,KAAI,SAAW,IACfA,KAAI,QAAU,iBACdA,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO,KACXA,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO/B,EAAkB,GAC7BjE,GAAE,YAAc,SAASwG,GACxBzH,EAASsb,kBACPhQ,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,QAC5BwC,KAAMwS,EAAGtX,EAAE6D,OAAQpG,EAAgB,OAItCjE,GAAE,YAAc,SAASwG,GACtBzH,EAASmG,kBACLmF,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAGnF,GACxBtC,EAASsF,iBACLgG,MAAO7D,EAAE6D,MACTqQ,MAAOlU,EAAEkU,OAAS,WAClB/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAItC0N,EAAE5Q,UAAS,s
BACRjD,KAAKgc,GACLnS,aACA1F,SAASA,GACTd,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SACrCrE,KAAI,KAAO,SAASQ,GAAK,MAAOsX,GAAGtX,EAAE6D,SAExC1E,EAAKC,UAAS,aACT5F,GAAE,YAAc,SAASwG,EAAEnF,GACxB,GAAIqZ,GAAQ4C,EAAYjc,IAAMud,EAAmBvd,EACjDtC,GAASsb,kBACLhQ,MAAO7D,EACPkU,MAAOA,EACP/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,YAAc,WACbjB,EAASmG,kBACLmF,MAAO+S,EAAS,GAChB1C,MAAOkD,EAAc,IAAM,WAC3BjT,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAEnF,GACvB,GAAIqZ,GAAQ4C,EAAYjc,IAAMud,EAAmBvd,EACjDtC,GAASsF,iBACLgG,MAAO7D,EACPkU,MAAOA,EACP/P,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,cAKrC3H,EAlQX,GAAIyC,IAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C6S,EAAS,OACTf,GAAU,EACVoH,EAAS,SAASvW,GAAK,MAAOA,GAAEuW,QAChCE,EAAU,SAASzW,GAAK,MAAOA,GAAEyW,QAAUzW,EAAEyW,YAC7CE,EAAc,SAAS3W,GAAK,MAAOA,GAAE2W,YAAc3W,EAAE2W,aAAe,IACpEE,EAAW,SAAS7W,GAAK,MAAOA,GAAE6W,UAClCE,EAAc,SAAS/W,GAAK,MAAOA,GAAE+W,YAAc/W,EAAE+W,gBACrDE,EAAe,SAASjX,GAAK,MAAOA,GAAEiX,aAAejX,EAAEiX,iBACvDE,EAAmB,SAASnX,GAAK,MAAOA,GAAEmX,i
BAAmBnX,EAAEmX,qBAC/DE,EAAgB,SAASrX,GAAK,MAAOA,GAAEqX,cAAgBrX,EAAEqX,kBACzDI,GAAU,GACVxY,EAAQ,IACRC,EAAS,GACTH,EAAY,KACZsR,EAAa,KACblM,EAAQtM,EAAGG,MAAMsQ,UAAQ,YACzB/P,EAAWF,GAAGE,SAAQ,mBAAqB,kBAAmB,oBAC9D6f,GAAqB,UAAY,OAAQ,WACzCP,GAAwB,MAAQ,MAAO,OACvCvX,EAAW,GAoRjB,OA/BA3F,GAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB0P,QAAczP,IAAK,WAAW,MAAOyP,IAAUxP,IAAK,SAASvG,GAAG+V,EAAO/V,IACvEiW,SAAc3P,IAAK,WAAW,MAAO2P,IAAW1P,IAAK,SAASvG,GAAGiW,EAAQjW,IACzEqW,UAAW/P,IAAK,WAAW,MAAO+P,IAAY9P,IAAK,SAASvG,GAAGqW,EAASrW,IACxEiX,QAAc3Q,IAAK,WAAW,MAAO2Q,IAAU1Q,IAAK,SAASvG,GAAGiX,EAAOjX,IACvEvB,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QAAY4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACrE6P,YAAgBvJ,IAAK,WAAW,MAAOuJ,IAActJ,IAAK,SAASvG,GAAG6P,EAAW7P,IACjFF,UAAcwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAAGF,EAASE,IAG3EpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IA
C3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D6S,QAASpJ,IAAK,WAAW,MAAOoJ,IAAUnJ,IAAK,SAASvG,GACpD0P,EAAS1P,EACT2O,EAAoB,SAAVe,GAA+B,UAAVA,IAEnC/L,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,OAIlC3I,EAAGG,MAAMqP,YAAY1M,GACdA,GChTX9C,EAAGI,OAAOogB,YAAc,WACpB,YA2BA,SAAS1d,GAAMsB,GAuHX,MAtHAA,GAAUC,KAAK,SAAS8D,EAAGnF,GACvB,GAAIkE,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,EAEjB,IAAIvB,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAOnD,IAJA/T,EAAMqR,OAAS,WAAarR,EAAMsB,IAClCtB,EAAMoE,UAAYnG,MAGboH,IAAMuW,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAE5B,MADAhD,GAAGG,MAAM4W,OAAOjU,EAAOoE,GAChBpE,CAEPoE,GAAUK,UAAS,cAAec,QAGtC,IAAIoW,GAASC,EAAOtd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YACjDG,EAAUC,EAAQxd,KAAKL,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YACnDO,EAAWC,EAAS5d,KAAKL
,KAAMoH,EAAGnF,GAAG7B,QAAQyU,KAAKpV,GAAGge,YAGrDlX,EAAOJ,EAAUK,UAAS,4BAA6BjD,MAAM6D,IAC7DX,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,+BACnD6V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,aAEjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAGvE,IAAIga,GAAKjf,GAAG8H,MAAMC,SACb5B,QAAQ,EAAG+C,KAAKL,IAAIoV,EAAO,GAAKE,EAAQ,IAAM,EAAII,EAAS,MAC3DtY,MAAM6Q,GAAW3R,EAAgB,IAAM,EAAGA,IAG3C8a,EAAK1f,KAAK8e,WAAarf,GAAG8H,MAAMC,SAC/B5B,QAAQ,EAAGoD,EAAAA,IACXtD,MAAMgZ,EAAGhZ,QAGd1F,MAAK8e,UAAYJ,CAEjB,IAGIiB,GAAQlD,EAAOrW,OAAM,cAAeO,OAAM,KACzCC,KAAI,cAAgB,OACpBA,KAAI,YAAc,iBAAmBN,EAAS9B,EAAOE,IAAMF,EAAOsR,QAAU,EAAI,IACrF6J,GAAMhZ,OAAM,QACPC,KAAI,QAAU,YACduK,KAAK,SAAS/J,GAAK,MAAOA,GAAEuY,QAEjCA,EAAMhZ,OAAM,QACPC,KAAI,QAAU,eACdA,KAAI,KAAO,OACXuK,KAAK,SAAS/J,GAAK,MAAOA,GAAEwY,WAEjC1C,EACK7W,MAAMzB,GACN0B,OAAOzB,EAEZ,IAAIgb,GAAazI,EAAEhR,OAAM,iBACzB3G,IAAG2N,WAAWyS,GAAYxf,KAAK6c,EAG/B,IAAItR,GAAS6L,GAAciH,EAAGjH,WAAY7S,EAAiB,KAGv
Dkb,EAAO1I,EAAE5Q,UAAS,aACjBjD,KAAKmb,EAAGrH,MAAOA,EAAQA,EAASzS,EAAiB,IAAO,SAASwC,GAC9D,MAAOpH,MAAK8Q,aAAelF,EAAOxE,KAItC2Y,EAAYD,EAAKpZ,QAAQC,OAAM,KAC9BC,KAAI,QAAU,WACdA,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesY,EAAGtY,GAAK,QAC9DsC,MAAK,UAAY,KAEtBqW,GAAUpZ,OAAM,QACXC,KAAI,KAAO/B,GACX+B,KAAI,KAAyB,EAAlB/B,EAAsB,GAEtCkb,EAAUpZ,OAAM,QACXC,KAAI,cAAgB,UACpBA,KAAI,KAAO,OACXA,KAAI,IAAwB,EAAlB/B,EAAsB,GAChCsM,KAAKvF,EAGV,IAAIoU,GAAavgB,GAAG2N,WAAW0S,GAC1B1S,aACA1F,SAASwV,EAAOxV,YAChBd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,GAAK,QAC9DsC,MAAK,UAAY,EAEtBsW,GAAW5Z,OAAM,QACZQ,KAAI,KAAO/B,GACX+B,KAAI,KAAyB,EAAlB/B,EAAsB,GAEtCmb,EAAW5Z,OAAM,QACZQ,KAAI,IAAwB,EAAlB/B,EAAsB,GAGrCpF,GAAG2N,WAAW0S,EAAKzY,QACd+F,aACA1F,SAASwV,EAAOxV,YAChBd,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAesX,EAAGtX,GAAK,QAC9DsC,MAAK,UAAY,MACjBpC,WAGT7H,GAAGwgB,MAAMC,QACFne,EA5IX,GAAImb,GAASje,EAAGI,OAAO6d,SACnB/d,EAAUF,EAAGI,OAAOF,UAEpBmY,EAAS,OACPf,GAAU,EACV/R,GAAUE,IAAK,EAAGqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,KAC/CkZ,EAAS,SAASvW,GAAK,MAAOA,GAAEuW,QA
ChCE,EAAU,SAASzW,GAAK,MAAOA,GAAEyW,QAAUzW,EAAEyW,YAC7CI,EAAW,SAAS7W,GAAK,MAAOA,GAAE6W,UAClC5X,EAAQ,KACRC,EAAS,GACTmR,EAAa,KACbJ,EAAQ,KACRrB,EAAS,KACTrW,EAAWF,GAAGE,UA2LpB,OAxLAR,GACKuI,SAAS,GACT+C,eAAc,GAgInByS,EAAOvd,SAASiB,GAAE,2BAA6B,SAASoc,GACpDA,EAAW,QACPvR,IAAKuR,EAAI1B,MACTrQ,MAAO+R,EAAI/R,MACXM,MAAOyR,EAAIzR,OAEfpM,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7B+X,EAAOvd,SAASiB,GAAE,0BAA4B,SAASoc,GACnD7d,EAAQgG,QAAO,KAGnB+X,EAAOvd,SAASiB,GAAE,2BAA6B,SAASoc,GACpD7d,MAOJ4C,EAAMmb,OAASA,EACfnb,EAAMpC,SAAWA,EACjBoC,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB0P,QAAczP,IAAK,WAAW,MAAOyP,IAAUxP,IAAK,SAASvG,GAAG+V,EAAO/V,IACvEiW,SAAc3P,IAAK,WAAW,MAAO2P,IAAW1P,IAAK,SAASvG,GAAGiW,EAAQjW,IACzEqW,UAAW/P,IAAK,WAAW,MAAO+P,IAAY9P,IAAK,SAASvG,GAAGqW,EAASrW,IACxEvB,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QAAY4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACrE6P,YAAgBvJ,IAAK,WAAW,MAAOuJ,IAActJ,IAAK,SAASvG,GAAG6P,EAAW7P,IACjFyP,O
AAWnJ,IAAK,WAAW,MAAOmJ,IAASlJ,IAAK,SAASvG,GAAGyP,EAAMzP,IAClEoO,QAAY9H,IAAK,WAAW,MAAO8H,IAAU7H,IAAK,SAASvG,GAAGoO,EAAOpO,IAGrEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAsB5B,SAAb8E,EAAElD,IAAuBkD,EAAElD,IAASF,EAAOE,IAC3DF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D6S,QAASpJ,IAAK,WAAW,MAAOoJ,IAAUnJ,IAAK,SAASvG,GACpD0P,EAAS1P,EACT2O,EAAoB,SAAVe,GAA+B,UAAVA,MAIvCrY,EAAGG,MAAMkW,eAAevT,EAAOmb,GAC/Bje,EAAGG,MAAMqP,YAAY1M,GAEdA,GCnNX9C,EAAGI,OAAO8gB,eAAiB,WACvB,YAoCA,SAASpe,GAAMsB,GA4HX,MA3HAA,GAAUC,KAAK,SAASC,GACpB4C,EAAY1G,GAAG2G,OAAOpG,KACtB,IAAI4E,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,EAElEvF,GAAGG,MAAMsW,QAAQvP,EAGjB,IAAIia,GAAYxb,EAAiBrB,EAAK,GAAGuE,OAAOzG,OAAU,GAG1D0F,GAAEnB,OAAOqT,GAAWxZ,GAAGkf,OAAOpb,EAAK,GAAGuE,OAAO4L,IAAIwF,GAAMvY,OAAOke,KAE1DwB,EACAtZ,EAAErB,
MAAMyT,IAA4B,GAAjBvU,EAAsBrB,EAAK,GAAGuE,OAAOzG,OAAQuD,GAAkBrB,EAAK,GAAGuE,OAAOzG,OAAS,IAAOkC,EAAK,GAAGuE,OAAOzG,SAEhI0F,EAAErB,MAAMyT,IAAW,EAAIiH,EAAW,EAAGxb,EAAiBwb,EAAW,EAAI,IAEzE5R,EAAE5I,OAAOyT,IACD5Z,GAAGgJ,IAAIlF,EAAK,GAAGuE,OAAO4L,IAAI4M,GAAQ3f,OAAO4f,IACzC9gB,GAAG6I,IAAI/E,EAAK,GAAGuE,OAAO4L,IAAI8M,GAAS7f,OAAO4f,MAEhD7a,MAAM0U,IAAWvV,EAAiB,IAGhCkC,EAAEnB,SAAS,KAAOmB,EAAEnB,SAAS,KAC7BmB,EAAEnB,SAAS,GACPmB,EAAEnB,QAAQmB,EAAEnB,SAAS,GAAqB,IAAhBmB,EAAEnB,SAAS,GAAWmB,EAAEnB,SAAS,GAAqB,IAAhBmB,EAAEnB,SAAS,KACzEmB,EAAEnB,QAAM,GAAK,KAEnB4I,EAAE5I,SAAS,KAAO4I,EAAE5I,SAAS,KAC7B4I,EAAE5I,SAAS,GACP4I,EAAE5I,QAAQ4I,EAAE5I,SAAS,GAAqB,IAAhB4I,EAAE5I,SAAS,GAAW4I,EAAE5I,SAAS,GAAqB,IAAhB4I,EAAE5I,SAAS,KACzE4I,EAAE5I,QAAM,GAAK,IAGvB,IAAIW,GAAO9G,GAAG2G,OAAOpG,MAAMwG,UAAS,+BAAgCjD,MAAMA,EAAK,GAAGuE,SAC9ErB,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,kCACnD8V,EAAYjW,EAAUE,OAAM,QAC5B8V,EAAShW,EAAUE,OAAM,KACzByQ,EAAI7Q,EAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,YAEjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,
KAAO,IAAMD,EAAOE,IAAM,KAEvEyB,EACKvF,GAAE,QAAU,SAASwG,EAAEnF,GACpBtC,EAAS8gB,YACLld,KAAM6D,EACNiB,MAAOpG,EACPiK,IAAKzM,GAAGuE,MACRwF,GAAIA,MAIhBkT,EAAU/V,OAAM,YACXC,KAAI,KAAO,sBAAwB4C,GACnC7C,OAAM,QAEXJ,EAAKH,OAAM,uBAA0BoD,EAAK,SACrC5C,KAAI,QAAUhC,GACdgC,KAAI,SAAW/B,GAEpBuS,EAAKxQ,KAAI,YAAc8Z,EAAW,2BAA6BlX,EAAK,IAAM,GAE1E,IAAI6N,GAAQ9Q,EAAKH,OAAM,aAAcI,UAAS,YACzCjD,KAAK,SAAS6D,GAAK,MAAOA,IAC/BiQ,GAAMhQ,OAAOC,QAEb,IAAIqZ,GAAatJ,EAAM3Q,QAAQC,OAAM,IAGrC0Q,GACKzQ,KAAI,QAAU,SAASQ,EAAGnF,EAAGwY,GAAK,OAAQmG,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAK,mBAAqB,oBAAsB,YAAcwY,EAAI,IAAMxY,GAEvI0e,GAAWha,OAAM,QACxBC,KAAI,QAAU,wBACdA,KAAI,YAAc,SAASQ,EAAGnF,GAAK,MAAO,aAAe8E,EAAEmS,EAAK9R,EAAGnF,IAAM,QACzE2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAEgS,EAAQpZ,EAAGnF,MAChD2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAE8R,EAAOlZ,EAAGnF,MAExC0e,EAAWha,OAAM,QACxBC,KAAI,QAAU,gCACdA,KAAI,YAAc,SAASQ,EAAGnF,GAC3B,MAAO,cAAgB8E,EAAEmS,EAAK9R,EAAGnF,IAAMme,EAAS,GAAK,KAClD5R,EAAEsS,EAAK1Z,EA
AGnF,KAAO2e,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAMuM,EAAEqS,EAASzZ,EAAGnF,IAAMuM,EAAEoS,EAAQxZ,EAAGnF,IAAO,IAC5F,MAEL2E,KAAI,IAAM,GACVA,KAAI,IAAM,GACVA,KAAI,QAAUwZ,GACdxZ,KAAI,SAAW,SAASQ,EAAGnF,GACxB,GAAI8e,GAAOH,EAAQxZ,EAAGnF,GAClB+e,EAAQH,EAASzZ,EAAGnF,EACxB,OAAO8e,GAAOC,EAAQxS,EAAEwS,GAASxS,EAAEuS,GAAQvS,EAAEuS,GAAQvS,EAAEwS,IAG/D3J,GAAMjR,OAAM,yBAA0BgH,aACjCxG,KAAI,YAAc,SAASQ,EAAGnF,GAAK,MAAO,aAAe8E,EAAEmS,EAAK9R,EAAGnF,IAAM,QACzE2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAEgS,EAAQpZ,EAAGnF,MAChD2E,KAAI,KAAO,GACXA,KAAI,KAAO,SAASQ,EAAGnF,GAAK,MAAOuM,GAAE8R,EAAOlZ,EAAGnF,MAEpDoV,EAAMjR,OAAM,yBAA0BgH,aACjCxG,KAAI,YAAc,SAASQ,EAAGnF,GAC3B,MAAO,cAAgB8E,EAAEmS,EAAK9R,EAAGnF,IAAMme,EAAS,GAAK,KAClD5R,EAAEsS,EAAK1Z,EAAGnF,KAAO2e,EAAQxZ,EAAGnF,GAAK4e,EAASzZ,EAAGnF,GAAMuM,EAAEqS,EAASzZ,EAAGnF,IAAMuM,EAAEoS,EAAQxZ,EAAGnF,IAAO,IAC5F,MAEL2E,KAAI,IAAM,GACVA,KAAI,IAAM,GACVA,KAAI,QAAUwZ,GACdxZ,KAAI,SAAW,SAASQ,EAAGnF,GACxB,GAAI8e,GAAOH,EAAQxZ,EAAGnF,GAClB+e,EAAQH,EAASzZ,EAAGnF,EACxB,
OAAO8e,GAAOC,EAAQxS,EAAEwS,GAASxS,EAAEuS,GAAQvS,EAAEuS,GAAQvS,EAAEwS,OAI5Djf,EA1JX,GAIMoE,GAeA8S,EACAI,EACAF,EACAiB,EAtBF5V,GAAUE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,GAC3C4B,EAAQ,KACRC,EAAS,KACTkD,EAAKb,KAAKwB,MAAsB,IAAhBxB,KAAKyB,UAErBrD,EAAItH,GAAG8H,MAAMC,SACbgH,EAAI/O,GAAG8H,MAAMC,SACb0R,EAAO,SAAS9R,GAAK,MAAOA,GAAEL,GAC9B+Z,EAAO,SAAS1Z,GAAK,MAAOA,GAAEoH,GAC9BoS,EAAU,SAASxZ,GAAK,MAAOA,GAAE2Z,MACjCF,EAAW,SAASzZ,GAAK,MAAOA,GAAE4Z,OAClCR,EAAU,SAASpZ,GAAK,MAAOA,GAAE6Z,MACjCX,EAAS,SAASlZ,GAAK,MAAOA,GAAE8Z,KAChCrC,KACA0B,KACAF,GAAc,EACdK,GAAW,EACXnV,EAAQtM,EAAGG,MAAMuQ,eACjBwR,GAAc,EAKdxhB,EAAWF,GAAGE,SAAQ,cAAgB,cAAe,YAAa,aAAc,eAAgB,kBAAmB,mBAAoB,kBAAmB,mBAiMhK,OAzDAoC,GAAMqf,eAAiB,SAASC,EAAYC,GACxCvf,EAAMwf,kBACNpb,EAAUC,OAAM,iCAAoCib,GAC/C1X,QAAO,QAAU2X,IAI1Bvf,EAAMwf,gBAAkB,WACpBpb,EAAUC,OAAM,qCACXuD,QAAO,SAAU,IAQ1B5H,EAAMpC,SAAWA,EACjBoC,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAW6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IAClEtB,QA
AW4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACpEvC,QAAW6I,IAAK,WAAW,MAAOnH,IAAKoH,IAAK,SAASvG,GAAGb,EAAEa,IAC1DuS,QAAWjM,IAAK,WAAW,MAAOM,IAAKL,IAAK,SAASvG,GAAG4G,EAAE5G,IAC1DqR,SAAW/K,IAAK,WAAW,MAAO+K,IAAW9K,IAAK,SAASvG,GAAGqR,EAAQrR,IACtEyR,SAAWnL,IAAK,WAAW,MAAOmL,IAAWlL,IAAK,SAASvG,GAAGyR,EAAQzR,IACtEuR,QAAWjL,IAAK,WAAW,MAAOiL,IAAUhL,IAAK,SAASvG,GAAGuR,EAAOvR,IACpEwS,QAAWlM,IAAK,WAAW,MAAOkM,IAAUjM,IAAK,SAASvG,GAAGwS,EAAOxS,IACpEiX,QAAW3Q,IAAK,WAAW,MAAO2Q,IAAU1Q,IAAK,SAASvG,GAAGiX,EAAOjX,IACpE2Y,QAAWrS,IAAK,WAAW,MAAOqS,IAAUpS,IAAK,SAASvG,GAAG2Y,EAAO3Y,IACpEyY,SAAWnS,IAAK,WAAW,MAAOmS,IAAWlS,IAAK,SAASvG,GAAGyY,EAAQzY,IACtE8Y,UAAWxS,IAAK,WAAW,MAAOwS,IAAYvS,IAAK,SAASvG,GAAG8Y,EAAS9Y,IACxE4B,IAAW0E,IAAK,WAAW,MAAO1E,IAAM2E,IAAK,SAASvG,GAAG4B,EAAG5B,IAC5DuZ,aAAcjT,IAAK,WAAW,MAAOiT,IAAehT,IAAK,SAASvG,GAAGuZ,EAAYvZ,IAEjFb,GAAQmH,IAAK,WAAW,MAAOgL,IAAQ/K,IAAK,SAASvG,GAAGsR,EAAKtR,IAC7D4G,GAAQN,IAAK,WAAW,MAAO4S,IAAQ3S,IAAK,SAASvG,GAAGkZ,EAAKlZ,IAC7DmZ,MAAQ7S,IAAK,WAAW,MAAO0S,MAAazS
,IAAK,SAASvG,GAAGgZ,EAAQhZ,IACrEoZ,OAAQ9S,IAAK,WAAW,MAAO2S,MAAc1S,IAAK,SAASvG,GAAGiZ,EAASjZ,IACvEqZ,MAAQ/S,IAAK,WAAW,MAAOsS,IAAWrS,IAAK,SAASvG,GAAG4Y,EAAQ5Y,IACnEsZ,KAAQhT,IAAK,WAAW,MAAOoS,IAAUnS,IAAK,SAASvG,GAAG0Y,EAAO1Y,IAGjEpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACpDpD,EAAOE,IAAqB5B,QAAZ8E,EAAElD,IAAsBkD,EAAElD,IAASF,EAAOE,IAC1DF,EAAOuR,MAAqBjT,QAAZ8E,EAAEmO,MAAsBnO,EAAEmO,MAASvR,EAAOuR,MAC1DvR,EAAOsR,OAAqBhT,QAAZ8E,EAAEkO,OAAsBlO,EAAEkO,OAAStR,EAAOsR,OAC1DtR,EAAOC,KAAqB3B,QAAZ8E,EAAEnD,KAAsBmD,EAAEnD,KAASD,EAAOC,OAE9D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,OAIlC3I,EAAGG,MAAMqP,YAAY1M,GACdA,GC/NX9C,EAAGI,OAAOmiB,oBAAsB,WAC5B,YAsFA,SAASzf,GAAMsB,GAsbX,MArbAqO,GAAYW,QACZX,EAAYrS,OAAOoiB,GACftF,GAAWzK,EAAYrS,OAAO+c,GAC9BC,GAAW3K,EAAYrS,OAAOid,GAClCjZ,EAAUC,KAAK,SAASC,GA0CpB,QAASme,GAAUta,EAAEnF,GACjBxC,GAAG2G,OAAOrE,EAAMoE,WACXuD,MAAK,SAAW,aAGzB,QAASiY,GAASva,EAAEnF,GAChBoG,EAAMtB,EAAItH,GAAGuE,MAAM+C,EACnBsB,EAAMpG,EAAI0G,KAAK6E,MAAM
oU,EAAG/b,OAAOwC,EAAMtB,IACrC8a,IAGJ,QAASC,GAAQ1a,EAAEnF,GACfxC,GAAG2G,OAAOrE,EAAMoE,WACXuD,MAAK,SAAW,QAGrBsH,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAgPzB,QAAS6Q,KACLG,GACKze,MAAM8E,GAIX,IAAI4Z,GAAclgB,EAAM2F,UACxB3F,GAAM2F,SAAS,GACf3F,EAAMqR,SACNrR,EAAM2F,SAASua,GAnTnB,GAAI9b,GAAY1G,GAAG2G,OAAOpG,KAC1Bf,GAAGG,MAAMsW,QAAQvP,GACjBA,EAAUwD,QAAO,YAAeH,GAAI,EACpC,IAEI5E,GAAiB3F,EAAGG,MAAMwF,eAAeyB,EAAOF,EAAW3B,GAC3DK,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,EAkBlE,IAhBAzC,EAAMqR,OAAS,WACM,IAAb1L,EACAvB,EAAU9F,KAAK0B,GAEfoE,EAAUiH,aAAa1F,SAASA,GAAUrH,KAAK0B,IAEvDA,EAAMoE,UAAYnG,KAElBgR,EACKmC,OAAO+O,EAAY3e,GAAOxB,EAAMqR,QAChCH,OAAOkP,EAAY5e,IACnB6P,SAGLpC,EAAM8L,SAAWvZ,EAAKmQ,IAAI,SAAStM,GAAK,QAASA,EAAE0V,YAE9CsF,EAAc,CACf,GAAI3W,EACJ2W,KACA,KAAK3W,IAAOuF,GACJA,EAAMvF,YAAgBtL,OACtBiiB,EAAa3W,GAAOuF,EAAMvF,GAAKrL,MAAM,GAErCgiB,EAAa3W,GAAOuF,EAAMvF,GAItC,GAAI4W,GAAY5iB,GAAG6iB,SAASC,OACvB3hB,GAAE,YAAc8gB,GAChB9gB,GAAE,OAAS+gB,GACX/gB,GAAE,UAAYkhB,EAwBnB,MAAKve,GAASA,EAAKlC,QAAWkC,EA
AKmI,OAAO,SAAStE,GAAK,MAAOA,GAAEU,OAAOzG,SAAUA,QAE9E,MADApC,GAAGG,MAAM4W,OAAOjU,EAAOoE,GAChBpE,CASX,IAPIoE,EAAUK,UAAS,cAAec,SAItCP,EAAI0a,EAAMpc,SACVmJ,EAAIiT,EAAMtH,SAELqI,EAsBDf,EAAMpI,QAAQ,UAtBH,CACX,GAAIoJ,GAAgBlf,EACfmI,OAAO,SAASL,GAAU,OAAQA,EAAOyR,WACzCpJ,IAAI,SAASrI,EAAOpJ,GACjB,GAAIygB,GAAgBjjB,GAAGkf,OAAOtT,EAAOvD,OAAQ2Z,EAAMjT,IAKnD,OAFIkU,GAAc,IAAK,MAAMA,EAAc,IAAK,OAGvCA,EAAc,GAAKA,EAAc,KAAO,EAAIA,EAAc,KAC1DA,EAAc,GAAKA,EAAc,KAAO,EAAIA,EAAc,OAIvEC,GACAljB,GAAGgJ,IAAIga,EAAe,SAASrb,GAAK,MAAOA,GAAE,KAC7C3H,GAAG6I,IAAIma,EAAe,SAASrb,GAAK,MAAOA,GAAE,KAGjDqa,GAAMpI,QAAQsJ,GAKlBf,EAAGhc,QAAQ,EAAGrC,EAAK,GAAGuE,OAAOzG,OAAS,IACjCqE,OAAO,EAAGd,IACV4X,OAAM,EAEX,IAAIjZ,GAAOqf,EAASva,EAAMpG,EAAGsB,GAGzBsf,EAA2B,EAA4B,OAAS,MAChEtc,EAAOJ,EAAUK,UAAS,+BAAgCjD,MAAMA,IAChEkZ,EAASlW,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,kCAAkCD,OAAM,KACxFyQ,EAAI7Q,EAAKH,OAAM,IA+BnB,IA7BAqW,EAAO9V,OAAM,KAAMC,KAAI,QAAU,kBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAAgB8C,MAAK,iBAAgB,QACtE+S,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gB
ACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,gBAAgB8C,MAAK,iBAAkBmZ,GACxEpG,EAAO9V,OAAM,KAAMC,KAAI,QAAU,mBAAmB8C,MAAK,iBAAgB,QACzE+S,EAAO9V,OAAM,KAAMC,KAAI,QAAU,iBACjC6V,EAAO9V,OAAM,KAAMC,KAAI,QAAU,mBAG5Bkc,GAGDC,EAAO1c,MAAMzB,GAEbwS,EAAEhR,OAAM,kBACHyW,MAAMtZ,GACNlD,KAAK0iB,GAELC,GAAaD,EAAOzc,WAAa9B,EAAOE,MACzCF,EAAOE,IAAMqe,EAAOzc,SACpBzB,EAAkB5F,EAAGG,MAAMyF,gBAAgByB,EAAQH,EAAW3B,IAGlE4S,EAAEhR,OAAM,kBACHQ,KAAI,YAAc,gBAAmBpC,EAAOE,IAAK,MAdtD0S,EAAEhR,OAAM,kBAAmBI,UAAS,KAAMc,SAkBzC2b,EAEE,CACH,GAAIC,KACEzX,IAAK,kBAAmBqR,UAAW0F,GAGzCW,GACK9c,MAAM,KACNkF,OAAK,OAAU,OAAQ,SACvB6X,YAAW,GACX5e,QAAQE,IAAK,EAAGqR,MAAO,EAAGD,OAAQ,EAAGrR,KAAM,KAGhD2S,EAAEhR,OAAM,oBACHyW,MAAMqG,GACNtc,KAAI,YAAc,gBAAmBpC,EAAOE,IAAK,KACjDrE,KAAK8iB,OAhBT/L,GAAEhR,OAAM,oBAAqBI,UAAS,KAAMc,QAmBjDf,GAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,KAEnEiY,GACAvF,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,aAAehC,EAAiB,MAI3D,IAAIye,GAAe9f,EAAKmI,OAAO,SAAStE,GAAK,MAAOA,GAAEic,cAEtD9c,GAAKH,OAAM,iBAAkBkB,
SACzB+b,EAAahiB,QACbkF,EAAKI,OAAM,QAASC,KAAI,QAAU,gBAC7BA,KAAI,IAAMhC,EAAiB,GAC3BgC,KAAI,IAAM,UACV8C,MAAK,cAAgB,OACrByH,KAAKkS,EAAa3P,IAAI,SAAStM,GAAK,MAAOA,GAAEqE,MAAOoL,KAAI,MAAS,sDAItEyM,IACAC,EACKld,MAAMzB,GACN0B,OAAOzB,GACPL,QAAQC,KAAKD,EAAOC,KAAKC,IAAIF,EAAOE,MACpCmC,aAAaV,GACbd,OAAO0B,GACZR,EAAKH,OAAM,mBAAoB/F,KAAKkjB,IAGxC9G,EAAOrW,OAAM,kBACRO,OAAM,QAEXyQ,EAAEhR,OAAM,uBACHQ,KAAI,QAAUhC,GACdgC,KAAI,SAAW/B,GAEpB4c,EAEKjT,EAAE,SAASpH,GAAK,MAAOA,GAAEoc,QAAQhV,IACjCnI,MAAMzB,GACN0B,OAAOzB,GACP0G,MAAMhI,EAAKmQ,IAAI,SAAStM,EAAEnF,GACvB,MAAOmF,GAAEmE,OAASA,EAAMnE,EAAGnF,KAC5ByJ,OAAO,SAAStE,EAAEnF,GAAK,OAAQsB,EAAKtB,GAAG6a,WAAavZ,EAAKtB,GAAGohB,eAEnE,IAAII,GAAYrM,EAAEhR,OAAM,iBACnByW,MAAMtZ,EAAKmI,OAAO,SAAStE,GAAK,OAASA,EAAE0V,WAAa1V,EAAEic,eAE/DI,GAAUpjB,KAAKohB,GAGfle,EAAK2F,QAAQ,SAAS9B,EAAEnF,GACpBmF,EAAEsc,YAAczhB,GAGpB,IAAI0hB,GAAcpgB,EAAKmI,OAAO,SAAStE,GACnC,OAAQA,EAAE0V,YAAc8G,EAAQxc,KAGhCyc,EAAWzM,EAAEhR,OAAM,oBAAqBI,UAAS,QAChDjD,KAAKogB,EAAa,SAASvc,GAAK,MAAOA,GAAEqE,MAE1CqY,
GAAc,SAAS1c,GAEvB,GAAI2c,GAAOvV,EAAEoV,EAAQxc,GACrB,OAAW,GAAP2c,EAAiB,EACjBA,EAAOlf,EAAwBA,EAC5Bkf,EAGXF,GAASnd,QACJC,OAAM,QACN+C,MAAK,eAAgB,GACrBA,MAAK,mBAAkB,SACvBA,MAAK,SAAU,SAAUtC,EAAEnF,GACxB,MAAOwf,GAAMlW,QAAQnE,EAAEA,EAAEsc,eAE5B9c,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAOkd,IACXld,KAAI,KAAOkd,IAEhBD,EACKna,MAAK,iBAAkB,SAAStC,GAE7B,GAAI2c,GAAOvV,EAAEoV,EAAQxc,GACrB,OAAW,GAAP2c,GAAYA,EAAOlf,EAAwB,EACxC,IAEV+B,KAAI,KAAM,GACVA,KAAI,KAAMhC,GACVgC,KAAI,KAAOkd,IACXld,KAAI,KAAOkd,IAEhBD,EAASxc,OAAOC,QAGhB,IAAI0a,IAAYyB,EAAUjd,UAAS,iBAC9BjD,MAAM8E,GACX2Z,IAAUtb,QAAQC,OAAM,QAASC,KAAI,QAAU,gBAC1CA,KAAI,QAAU,GACdA,KAAI,IAAM,IACVA,KAAI,OAAS,OACbA,KAAI,eAAiB,IACrB8C,MAAK,iBAAgB,OACrBrJ,KAAKgiB,GAEVL,GACKpb,KAAI,YAAc,SAASQ,GAAK,MAAO,aAAewa,EAAGxa,EAAEnF,GAAK,QAChE2E,KAAI,SAAW/B,GAGhBsX,IACAC,EACK7U,MAAMR,GACNid,OAAQ/kB,EAAGG,MAAMuU,WAAW/O,EAAe,GAAIrB,IAC/CwZ,UAAUlY,EAAiB,GAEhCuS,EAAEhR,OAAM,iBACHQ,KAAI,YAAc,eAAiB4H,EAAE9I,QAAQ,GAAK,KACvD0R,EAAEhR,OAAM,iBACH/F,KAAK+b,IAGVC,IACAC,EACK/U,M
AAMiH,GACNwV,OAAQ/kB,EAAGG,MAAM2U,WAAWlP,EAAgB,GAAItB,IAChDwZ,UAAWnY,EAAgB,GAEhCwS,EAAEhR,OAAM,iBACH/F,KAAKic,IAmBdlF,EAAEhR,OAAM,uBACHxF,GAAE,QAAU,WACTyH,EAAMtB,EAAItH,GAAGiE,MAAM1D,MAAM,GACzBqI,EAAMpG,EAAI0G,KAAK6E,MAAMoU,EAAG/b,OAAOwC,EAAMtB,IAGrCiK,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAErB6Q,MAGRJ,EAAM9hB,SAASiB,GAAE,eAAiB,SAASC,GACvCwH,EAAMpG,EAAIpB,EAAEwgB,WACZhZ,EAAMtB,EAAI6a,EAAGvZ,EAAMpG,GAGnB+O,EAAM3I,MAAQA,EAAMpG,EACpBtC,EAASoiB,YAAY/Q,GAErB6Q,MAGJsB,EAASxjB,SAASiB,GAAE,cAAgB,SAASwG,EAAEnF,GAC3CmF,EAAE0V,UAAY1V,EAAE0V,SAChB0F,GAAYpb,EAAE0V,SAEd9L,EAAMwR,SAAWA,EACjB7iB,EAASoiB,YAAY/Q,GACrBjP,EAAMqR,WAGV2P,EAAOpjB,SAASiB,GAAE,cAAgB,SAASqjB,GACvC,IAAK,GAAIxY,KAAOwY,GACZjT,EAAMvF,GAAOwY,EAASxY,EAC1B9L,GAASoiB,YAAY/Q,GACrBjP,EAAMqR,WAGVmQ,EAAiB5jB,SAASiB,GAAE,mBAAqB,SAASC,GACtD4gB,EAAMF,iBACN,IAAI2C,GAAa7C,EAAY8C,EAAgBC,IAsB7C,IApBA7gB,EACKmI,OAAO,SAASL,EAAQpJ,GAErB,MADAoJ,GAAOqY,YAAczhB,GACboJ,EAAOyR,WAElB5T,QAAQ,SAASmC,EAAOpJ,GACrBof,EAAapiB,EAAG4I,kBAAkBwD,EAAOvD,OAA
QjH,EAAE0E,YAAaxD,EAAMgF,KACtE0a,EAAML,eAAenf,EAAGof,GAAY,EACpC,IAAI9S,GAAQlD,EAAOvD,OAAOuZ,EACL,oBAAV9S,KACgB,mBAAhB2V,KAA6BA,EAAc3V,GACxB,mBAAnB4V,KAAgCA,EAAiBpiB,EAAMsD,SAAStD,EAAMgF,IAAIwH,EAAM8S,KAC3F+C,EAAQ1hB,MACJ+I,IAAKJ,EAAOI,IACZR,MAAOlJ,EAAMyM,IAAID,EAAO8S,GACxB9V,MAAOA,EAAMF,EAAOA,EAAOqY,kBAKnCU,EAAQ/iB,OAAS,EAAG,CACpB,GAAIgjB,GAAStiB,EAAMoY,SAAStU,OAAOhF,EAAE+C,QACjC0gB,EAAe3b,KAAKC,IAAI7G,EAAMoY,SAASvU,SAAS,GAAK7D,EAAMoY,SAASvU,SAAS,IAC7EkD,EAAY,IAAOwb,EACnBrb,EAAmBhK,EAAG4J,kBAAkBub,EAAQ1Q,IAAI,SAAStM,GAAG,MAAOA,GAAE6D,QAAQoZ,EAAOvb,EACnE,QAArBG,IACAmb,EAAQnb,GAAkBqC,WAAY,GAG9C,GAAIiZ,GAASnI,EAAM3E,aAAa1V,EAAMgF,IAAImd,EAAY7C,GAAaA,EACnEkC,GAAiBpkB,QACZuL,eAAe,SAAStD,EAAEnF,GACvB,MAAOqa,GAAM7E,aAAarQ,KAE7B7D,MAEG0H,MAAOsZ,EACPlZ,OAAQ+Y,MAIhBb,EAAiBre,gBAAgBif,KAGrCZ,EAAiB5jB,SAASiB,GAAE,kBAAmB,SAASC,GACpD4gB,EAAMF,oBAIV5hB,EAASiB,GAAE,cAAgB,SAASC,GACN,mBAAfA,GAAEic,WACTvZ,EAAK2F,QAAQ,SAASmC,EAAOpJ,GACzBoJ,EAAOyR,SAAWjc,EAAEic,SAAS7a,KAGjC+O,EAAM8L,SAAWjc,EAAEic,UAGA,m
BAAZjc,GAAEwH,QACTA,EAAMpG,EAAIpB,EAAEwH,MACZA,EAAMtB,EAAI6a,EAAGvZ,EAAMpG,GAEnB+O,EAAM3I,MAAQxH,EAAEwH,MAEhB2Z,GACKze,MAAM8E,KAGW,mBAAfxH,GAAE2hB,WACTA,EAAW3hB,EAAE2hB,UAGjBzgB,EAAMqR,aAKd1B,EAAYS,UAAS,iCAEdpQ,EA2BX,QAAS6gB,GAAS4B,EAAKjhB,GAEnB,MADKkhB,KAAiBA,EAAkBhD,EAAMjT,KACvCjL,EAAKmQ,IAAI,SAASzM,EAAMhF,GAC3B,IAAKgF,EAAKa,OACN,MAAOb,EAEX,IAAIyd,GAAazd,EAAKa,OAAO0c,EAC7B,IAAkB,MAAdE,EACA,MAAOzd,EAEX,IAAIkB,GAAIsc,EAAgBC,EAAYF,EAGpC,QAAQ,IAAJrc,IAAawc,GAGb1d,EAAKoc,cAAe,EACbpc,IAGXA,EAAKoc,cAAe,EAEpBpc,EAAKa,OAASb,EAAKa,OAAO4L,IAAI,SAASnF,EAAO8S,GAE1C,MADA9S,GAAMiV,SAAUhV,GAAOiW,EAAgBlW,EAAO8S,GAAclZ,IAAM,EAAIA,IAC/DoG,IAGJtH,KA5jBf,GAqBMF,GACAyH,EAtBFiT,EAAQxiB,EAAGI,OAAO4H,OAChBmV,EAAQnd,EAAGI,OAAO8X,OAClBmF,EAAQrd,EAAGI,OAAO8X,OAClB4L,EAAS9jB,EAAGI,OAAO0jB,SACnBI,EAAWlkB,EAAGI,OAAO0jB,SACrBQ,EAAmBtkB,EAAGkE,uBACtBhE,EAAUF,EAAGI,OAAOF,UAGtBqF,GAAUE,IAAK,GAAIqR,MAAO,GAAID,OAAQ,GAAIrR,KAAM,IAC9Cue,EAAY,KACZzX,EAAQtM,EAAGG,MAAMuQ,eACjBtJ,EAAQ,KACRC,EAAS,KACTwc,GAAa,EACb3G,GAAY,EAC
ZE,GAAY,EACZM,GAAkB,EAClBsG,GAAe,EACfK,GAA0B,EAC1Bd,GAAW,EAGXhZ,EAAKiY,EAAMjY,KACXwH,EAAQ/R,EAAGG,MAAM4R,QACjBoR,EAAe,KACfpM,EAAS,KACT4N,EAAU,SAASxc,GAAK,MAAOA,GAAEwc,SACjCjkB,EAAWF,GAAGE,SAAQ,cAAgB,cAAe,aAErD+H,EAAW,IACXid,GAAe,CAGrB3T,GAAM3I,MAAQ,EACd2I,EAAMwR,SAAWA,EAEjBpG,EAAM9E,OAAM,UAAWW,YAAY,GACnCqE,EAAMhF,OAAM,EAAqB,QAAU,QAE3CnY,EAAQuL,eAAe,SAAStD,EAAGnF,GAC/B,MAAOqa,GAAM7E,aAAarQ,EAAGnF,KAC9B0I,gBAAgB,SAASvD,EAAGnF,GAC3B,MAAOma,GAAM3E,aAAarQ,EAAGnF,KAGjCkhB,EAASyB,aAAY,EAMrB,IAAIhD,GAAKniB,GAAG8H,MAAMC,SACZa,GAASpG,EAAG,EAAG8E,EAAG,GAClB2K,EAAczS,EAAGG,MAAMsS,YAAY/R,EAAU+H,GAG/Cya,EAAc,SAAS5e,GACvB,MAAO,YACH,OACI3B,OAAQ2B,EAAKmQ,IAAI,SAAStM,GAAK,OAAQA,EAAE0V,WACzCzU,MAAOA,EAAMpG,EACbugB,SAAUA,KAKlBN,EAAc,SAAS3e,GACvB,MAAO,UAASyN,GACQlO,SAAhBkO,EAAM3I,QACNA,EAAMpG,EAAI+O,EAAM3I,OACGvF,SAAnBkO,EAAMwR,WACNA,EAAWxR,EAAMwR,UACA1f,SAAjBkO,EAAMpP,QACN2B,EAAK2F,QAAQ,SAASmC,EAAOpJ,GACzBoJ,EAAOyR,UAAY9L,EAAMpP,OAAOK,MAkchDwf,GAAM9hB,SAASiB,GAAE,2BAA6B,SAASoc,GACnD,GAAIzO,IACAxH,EAAGh
F,EAAMgF,IAAIiW,EAAIzO,OACjBC,EAAGzM,EAAMyM,IAAIwO,EAAIzO,OACjBhD,MAAOyR,EAAIzO,MAAMhD,MAErByR,GAAIzO,MAAQA,EACZpP,EAAQoE,KAAKyZ,GAAK7X,QAAO,KAG7Bsc,EAAM9hB,SAASiB,GAAE,0BAA4B,SAASoc,GAClD7d,EAAQgG,QAAO,IAOnB,IAAIsf,GAAkB,IAqGtB,OA/DA1iB,GAAMpC,SAAWA,EACjBoC,EAAM0f,MAAQA,EACd1f,EAAMghB,OAASA,EACfhhB,EAAMohB,SAAWA,EACjBphB,EAAMqa,MAAQA,EACdra,EAAMua,MAAQA,EACdva,EAAMwhB,iBAAmBA,EACzBxhB,EAAMiP,MAAQA,EACdjP,EAAM5C,QAAUA,EAEhB4C,EAAM8L,QAAU5O,EAAGG,MAAM0O,YAAYhO,KAAKiC,GAE1CA,EAAMgM,SAAWC,OAAOC,WAEpB5H,OAAa6H,IAAK,WAAW,MAAO7H,IAAS8H,IAAK,SAASvG,GAAGvB,EAAMuB,IACpEtB,QAAa4H,IAAK,WAAW,MAAO5H,IAAU6H,IAAK,SAASvG,GAAGtB,EAAOsB,IACtE4a,UAAetU,IAAK,WAAW,MAAOsU,IAAYrU,IAAK,SAASvG,GAAG4a,EAAS5a,IAC5Eqb,cAAmB/U,IAAK,WAAW,MAAO+U,IAAgB9U,IAAK,SAASvG,GAAGqb,EAAarb,IACxFkb,YAAa5U,IAAK,WAAW,MAAO4U,IAAc3U,IAAK,SAASvG,GAAGkb,EAAWlb,IAC9Egc,SAAU1V,IAAK,WAAW,MAAO0V,IAAWzV,IAAK,SAASvG,GAAGgc,EAAQhc,IACrEwa,cAAkBlU,IAAK,WAAW,MAAOkU,IAAgBjU,IAAK,SAASvG,GAAGwa,EAAaxa,IACvFoO,QAAY9H,IAAK,WAAW,MAAO8H,IAAU7H,IA
AK,SAASvG,GAAGoO,EAAOpO,IACrEuU,WAAejO,IAAK,WAAW,MAAOiO,IAAahO,IAAK,SAASvG,GAAGuU,EAAUvU,IAC9EyU,WAAenO,IAAK,WAAW,MAAOmO,IAAalO,IAAK,SAASvG,GAAGyU,EAAUzU,IAC9E+c,cAAkBzW,IAAK,WAAW,MAAOyW,IAAgBxW,IAAK,SAASvG,GAAG+c,EAAa/c,IAGvFpD,QAAS0J,IAAK,WAAW,MAAO1J,IAAU2J,IAAK,SAASvG,GACtC9E,SAAV8E,EAAElD,MACFF,EAAOE,IAAMkD,EAAElD,IACfse,EAAYpb,EAAElD,KAElBF,EAAOuR,MAAsBjT,SAAb8E,EAAEmO,MAAuBnO,EAAEmO,MAASvR,EAAOuR,MAC3DvR,EAAOsR,OAAsBhT,SAAb8E,EAAEkO,OAAuBlO,EAAEkO,OAAStR,EAAOsR,OAC3DtR,EAAOC,KAAsB3B,SAAb8E,EAAEnD,KAAuBmD,EAAEnD,KAASD,EAAOC,OAE/D8G,OAAS2C,IAAK,WAAW,MAAO3C,IAAS4C,IAAK,SAASvG,GACnD2D,EAAQtM,EAAGG,MAAMsQ,SAAS9H,GAC1Bmb,EAAOxX,MAAMA,KAEjB+X,yBAA0BpV,IAAK,WAAW,MAAOoV,IAA2BnV,IAAK,SAASvG,GACtF0b,EAA0B1b,EACtBA,KAAM,IACN7F,EAAMof,aAAY,GAClBpf,EAAM8iB,YAAW,MAGzBlI,iBAAkBzO,IAAK,WAAW,MAAOyO,IAAmBxO,IAAK,SAASvG,GACtE+U,EAAkB/U,EAClB0U,EAAMhF,OAAQ,EAAM,QAAU,UAElC5P,UAAcwG,IAAK,WAAW,MAAOxG,IAAYyG,IAAK,SAASvG,GAC3DF,EAAWE,EACX6Z,EAAM/Z,SAASA,GACf0U,EAAM1U,SAASA,GACf4U,EAAM5U,SAASA,GACfgK,EA
AYW,MAAM3K,OAI1BzI,EAAGG,MAAMkW,eAAevT,EAAO0f,GAC/BxiB,EAAGG,MAAMqP,YAAY1M,GAEdA,GC3oBX9C,EAAGI,OAAOylB,YAAc,WACpB,YAmCA,SAAS/iB,GAAMsB,GAyKX,MAxKAqO,GAAYW,QACZhP,EAAUC,KAAK,SAASC,GACpB,GAAIqB,GAAiByB,EAAQ7B,EAAOC,KAAOD,EAAOuR,MAC9ClR,EAAkByB,EAAS9B,EAAOE,IAAMF,EAAOsR,MAEnD3P,GAAY1G,GAAG2G,OAAOpG,MACtBf,EAAGG,MAAMsW,QAAQvP,GAGjB5C,EAAK2F,QAAQ,SAASmC,EAAQpJ,GAC1BoJ,EAAOvD,OAAOoB,QAAQ,SAASqF,GAC3BA,EAAMlD,OAASpJ,KAMvB,IAAI8iB,GAAc9L,GAAWI,KACzB9V,EAAKmQ,IAAI,SAAStM,GACd,MAAOA,GAAEU,OAAO4L,IAAI,SAAStM,EAAEnF,GAC3B,OAAS8E,EAAGmS,EAAK9R,EAAEnF,GAAIuM,EAAGsS,EAAK1Z,EAAEnF,GAAI+iB,GAAI5d,EAAE4d,OAIvDje,GAAKnB,OAAOqT,GAAWxZ,GAAGmf,MAAMmG,GAAYrR,IAAI,SAAStM,GAAK,MAAOA,GAAEL,KAClEzB,WAAW6T,IAAW,EAAGvU,GAAiB,IAC/C4J,EAAK5I,OAAOyT,GAAW5Z,GAAGkf,OAAOlf,GAAGmf,MAAMmG,GAAYrR,IAAI,SAAStM,GAAK,MAAOA,GAAEoH,IAAK7N,OAAO4f,KAGzF0E,EAAYzW,EAAE9I,MAAM0U,IAAWvV,GAAmB2J,EAAE5I,SAAS,GAAK,EAAI,GAAK,GAAI4I,EAAE5I,SAAS,GAAK,EAAI,GAAK,IACvG4I,EAAE9I,MAAM0U,IAAWvV,EAAiB,IAGzC6a,EAAKA,GAAM3Y,EACXie,EAAKA,GAAMxW,E
AAEuK,OAAOrT,OAAO8I,EAAE,GAAGA,EAAE,IAGlC,IAAIjI,GAAOJ,EAAUK,UAAS,4BAA6BjD,MAAMA,IAC7DkD,EAAYF,EAAKG,QAAQC,OAAM,KAAMC,KAAI,QAAU,+BACnD6V,EAAShW,EAAUE,OAAM,IACrBJ,GAAKH,OAAM,IAEnBqW,GAAO9V,OAAM,KAAMC,KAAI,QAAU,aACjCL,EAAKK,KAAI,YAAc,aAAepC,EAAOC,KAAO,IAAMD,EAAOE,IAAM,IAGvE,IAAIwgB,GAAS3e,EAAKH,OAAM,cAAeI,UAAS,aAC3CjD,KAAK,SAAS6D,GAAK,MAAOA,IAAK,SAASA,GAAK,MAAOA,GAAEqE,KAC3DyZ,GAAOxe,QAAQC,OAAM,KAChB+C,MAAK,iBAAmB,MACxBA,MAAK,eAAiB,MAC3Bwb,EAAO7d,OACFoK,gBAAgBC,EAAa,4BAC7BhI,MAAK,iBAAmB,MACxBA,MAAK,eAAiB,MACtBpC,SACL4d,EACKte,KAAI,QAAU,SAASQ,EAAEnF,GAAK,MAAO,sBAAwBA,IAC7D0H,QAAO,QAAU,SAASvC,GAAK,MAAOA,GAAEsT,QAC7CwK,EACKzT,gBAAgBC,EAAa,uBAC7BhI,MAAK,iBAAmB,GACxBA,MAAK,eAAiB,IAE3B,IAAIyb,GAAOD,EAAO1e,UAAS,YACtBjD,KAAK,SAAS6D,GAAK,MAAOA,GAAEU,QACjCqd,GAAK9d,OAAOC,QAEZ,IAAI8d,GAAYD,EAAKze,QAAQC,OAAM,KAC9BC,KAAI,YAAc,SAASQ,EAAEnF,EAAEwY,GAC5B,MAAO,cAAgB1T,EAAEmS,EAAK9R,EAAEnF,IAAsB,IAAhB8E,EAAEpB,aAAsB,KAAO6I,EAAE,GAAK,MAE/E5N,GAAE,YAAc,SAASwG,EAAEnF,GACxBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU
,GACjChK,EAASsb,kBACL1X,KAAM6D,EACNiB,MAAOpG,EACPsJ,MAAO9L,GAAG2G,OAAOpG,MAAM0J,MAAK,YAGnC9I,GAAE,WAAa,SAASwG,EAAEnF,GACvBxC,GAAG2G,OAAOpG,MAAM2J,QAAO,SAAU,GACjChK,EAASsF,iBACL1B,KAAM6D,EACNiB,MAAOpG,EACPsJ,MAAO9L
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
deleted file mode 100644
index bf301b8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,304 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.NodeQueryResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link StorageContainerLocationProtocol} interface to the RPC server
- * implementing {@link StorageContainerLocationProtocolPB}.
- */
-@InterfaceAudience.Private
-public final class StorageContainerLocationProtocolClientSideTranslatorPB
- implements StorageContainerLocationProtocol, ProtocolTranslator, Closeable {
-
- /**
- * RpcController is not used and hence is set to null.
- */
- private static final RpcController NULL_RPC_CONTROLLER = null;
-
- private final StorageContainerLocationProtocolPB rpcProxy;
-
- /**
- * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
- *
- * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy
- */
- public StorageContainerLocationProtocolClientSideTranslatorPB(
- StorageContainerLocationProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- /**
- * Asks SCM where a container should be allocated. SCM responds with the set
- * of datanodes that should be used creating this container. Ozone/SCM only
- * supports replication factor of either 1 or 3.
- * @param type - Replication Type
- * @param factor - Replication Count
- * @param containerName - Name
- * @return
- * @throws IOException
- */
- @Override
- public Pipeline allocateContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, String
- containerName, String owner) throws IOException {
-
- Preconditions.checkNotNull(containerName, "Container Name cannot be Null");
- Preconditions.checkState(!containerName.isEmpty(), "Container name cannot" +
- " be empty");
- ContainerRequestProto request = ContainerRequestProto.newBuilder()
- .setContainerName(containerName)
- .setReplicationFactor(factor)
- .setReplicationType(type)
- .setOwner(owner)
- .build();
-
- final ContainerResponseProto response;
- try {
- response = rpcProxy.allocateContainer(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (response.getErrorCode() != ContainerResponseProto.Error.success) {
- throw new IOException(response.hasErrorMessage() ?
- response.getErrorMessage() : "Allocate container failed.");
- }
- return Pipeline.getFromProtoBuf(response.getPipeline());
- }
-
- public Pipeline getContainer(String containerName) throws IOException {
- Preconditions.checkNotNull(containerName,
- "Container Name cannot be Null");
- Preconditions.checkState(!containerName.isEmpty(),
- "Container name cannot be empty");
- GetContainerRequestProto request = GetContainerRequestProto
- .newBuilder()
- .setContainerName(containerName)
- .build();
- try {
- GetContainerResponseProto response =
- rpcProxy.getContainer(NULL_RPC_CONTROLLER, request);
- return Pipeline.getFromProtoBuf(response.getPipeline());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException {
- ListContainerRequestProto.Builder builder = ListContainerRequestProto
- .newBuilder();
- if (prefixName != null) {
- builder.setPrefixName(prefixName);
- }
- if (startName != null) {
- builder.setStartName(startName);
- }
- builder.setCount(count);
- ListContainerRequestProto request = builder.build();
-
- try {
- ListContainerResponseProto response =
- rpcProxy.listContainer(NULL_RPC_CONTROLLER, request);
- List<ContainerInfo> containerList = new ArrayList<>();
- for (OzoneProtos.SCMContainerInfo containerInfoProto : response
- .getContainersList()) {
- containerList.add(ContainerInfo.fromProtobuf(containerInfoProto));
- }
- return containerList;
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- /**
- * Ask SCM to delete a container by name. SCM will remove
- * the container mapping in its database.
- *
- * @param containerName
- * @throws IOException
- */
- @Override
- public void deleteContainer(String containerName)
- throws IOException {
- Preconditions.checkState(!Strings.isNullOrEmpty(containerName),
- "Container name cannot be null or empty");
- DeleteContainerRequestProto request = DeleteContainerRequestProto
- .newBuilder()
- .setContainerName(containerName)
- .build();
- try {
- rpcProxy.deleteContainer(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- /**
- * Queries a list of Node Statuses.
- *
- * @param nodeStatuses
- * @return List of Datanodes.
- */
- @Override
- public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState>
- nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName)
- throws IOException {
- // TODO : We support only cluster wide query right now. So ignoring checking
- // queryScope and poolName
- Preconditions.checkNotNull(nodeStatuses);
- Preconditions.checkState(nodeStatuses.size() > 0);
- NodeQueryRequestProto request = NodeQueryRequestProto.newBuilder()
- .addAllQuery(nodeStatuses)
- .setScope(queryScope).setPoolName(poolName).build();
- try {
- NodeQueryResponseProto response =
- rpcProxy.queryNode(NULL_RPC_CONTROLLER, request);
- return response.getDatanodes();
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- }
-
- /**
- * Notify from client that creates object on datanodes.
- * @param type object type
- * @param name object name
- * @param op operation type (e.g., create, close, delete)
- * @param stage object creation stage : begin/complete
- */
- @Override
- public void notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type type, String name,
- ObjectStageChangeRequestProto.Op op,
- ObjectStageChangeRequestProto.Stage stage) throws IOException {
- Preconditions.checkState(!Strings.isNullOrEmpty(name),
- "Object name cannot be null or empty");
- ObjectStageChangeRequestProto request =
- ObjectStageChangeRequestProto.newBuilder()
- .setType(type)
- .setName(name)
- .setOp(op)
- .setStage(stage)
- .build();
- try {
- rpcProxy.notifyObjectStageChange(NULL_RPC_CONTROLLER, request);
- } catch(ServiceException e){
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- /**
- * Creates a replication pipeline of a specified type.
- *
- * @param replicationType - replication type
- * @param factor - factor 1 or 3
- * @param nodePool - optional machine list to build a pipeline.
- * @throws IOException
- */
- @Override
- public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType
- replicationType, OzoneProtos.ReplicationFactor factor, OzoneProtos
- .NodePool nodePool) throws IOException {
- PipelineRequestProto request = PipelineRequestProto.newBuilder()
- .setNodePool(nodePool)
- .setReplicationFactor(factor)
- .setReplicationType(replicationType)
- .build();
- try {
- PipelineResponseProto response =
- rpcProxy.allocatePipeline(NULL_RPC_CONTROLLER, request);
- if (response.getErrorCode() ==
- PipelineResponseProto.Error.success) {
- Preconditions.checkState(response.hasPipeline(), "With success, " +
- "must come a pipeline");
- return Pipeline.getFromProtoBuf(response.getPipeline());
- } else {
- String errorMessage = String.format("create replication pipeline " +
- "failed. code : %s Message: %s", response.getErrorCode(),
- response.hasErrorMessage() ? response.getErrorMessage() : "");
- throw new IOException(errorMessage);
- }
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
- public ScmInfo getScmInfo() throws IOException {
- OzoneProtos.GetScmInfoRequestProto request =
- OzoneProtos.GetScmInfoRequestProto.getDefaultInstance();
- try {
- OzoneProtos.GetScmInfoRespsonseProto resp = rpcProxy.getScmInfo(
- NULL_RPC_CONTROLLER, request);
- ScmInfo.Builder builder = new ScmInfo.Builder()
- .setClusterId(resp.getClusterId())
- .setScmId(resp.getScmId());
- return builder.build();
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
-
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public void close() {
- RPC.stopProxy(rpcProxy);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
deleted file mode 100644
index 9ee1307..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/StorageContainerLocationProtocolPB.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.StorageContainerLocationProtocolService;
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager. This extends the
- * Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.ozone.protocol.StorageContainerLocationProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface StorageContainerLocationProtocolPB
- extends StorageContainerLocationProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
deleted file mode 100644
index f9a2c09..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocolPB;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
deleted file mode 100644
index 943be2a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkInputStream.java
+++ /dev/null
@@ -1,261 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.ByteBuffer;
-import java.util.Arrays;
-import java.util.List;
-
-import com.google.protobuf.ByteString;
-
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadChunkResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.XceiverClientManager;
-
-/**
- * An {@link InputStream} used by the REST service in combination with the
- * SCMClient to read the value of a key from a sequence
- * of container chunks. All bytes of the key value are stored in container
- * chunks. Each chunk may contain multiple underlying {@link ByteBuffer}
- * instances. This class encapsulates all state management for iterating
- * through the sequence of chunks and the sequence of buffers within each chunk.
- */
-public class ChunkInputStream extends InputStream implements Seekable {
-
- private static final int EOF = -1;
-
- private final String key;
- private final String traceID;
- private XceiverClientManager xceiverClientManager;
- private XceiverClientSpi xceiverClient;
- private List<ChunkInfo> chunks;
- private int chunkIndex;
- private long[] chunkOffset;
- private List<ByteBuffer> buffers;
- private int bufferIndex;
-
- /**
- * Creates a new ChunkInputStream.
- *
- * @param key chunk key
- * @param xceiverClientManager client manager that controls client
- * @param xceiverClient client to perform container calls
- * @param chunks list of chunks to read
- * @param traceID container protocol call traceID
- */
- public ChunkInputStream(String key, XceiverClientManager xceiverClientManager,
- XceiverClientSpi xceiverClient, List<ChunkInfo> chunks, String traceID) {
- this.key = key;
- this.traceID = traceID;
- this.xceiverClientManager = xceiverClientManager;
- this.xceiverClient = xceiverClient;
- this.chunks = chunks;
- this.chunkIndex = -1;
- // chunkOffset[i] stores offset at which chunk i stores data in
- // ChunkInputStream
- this.chunkOffset = new long[this.chunks.size()];
- initializeChunkOffset();
- this.buffers = null;
- this.bufferIndex = 0;
- }
-
- private void initializeChunkOffset() {
- int tempOffset = 0;
- for (int i = 0; i < chunks.size(); i++) {
- chunkOffset[i] = tempOffset;
- tempOffset += chunks.get(i).getLen();
- }
- }
-
- @Override
- public synchronized int read()
- throws IOException {
- checkOpen();
- int available = prepareRead(1);
- return available == EOF ? EOF :
- Byte.toUnsignedInt(buffers.get(bufferIndex).get());
- }
-
- @Override
- public synchronized int read(byte[] b, int off, int len) throws IOException {
- // According to the JavaDocs for InputStream, it is recommended that
- // subclasses provide an override of bulk read if possible for performance
- // reasons. In addition to performance, we need to do it for correctness
- // reasons. The Ozone REST service uses PipedInputStream and
- // PipedOutputStream to relay HTTP response data between a Jersey thread and
- // a Netty thread. It turns out that PipedInputStream/PipedOutputStream
- // have a subtle dependency (bug?) on the wrapped stream providing separate
- // implementations of single-byte read and bulk read. Without this, get key
- // responses might close the connection before writing all of the bytes
- // advertised in the Content-Length.
- if (b == null) {
- throw new NullPointerException();
- }
- if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
- }
- if (len == 0) {
- return 0;
- }
- checkOpen();
- int available = prepareRead(len);
- if (available == EOF) {
- return EOF;
- }
- buffers.get(bufferIndex).get(b, off, available);
- return available;
- }
-
- @Override
- public synchronized void close() {
- if (xceiverClientManager != null && xceiverClient != null) {
- xceiverClientManager.releaseClient(xceiverClient);
- xceiverClientManager = null;
- xceiverClient = null;
- }
- }
-
- /**
- * Checks if the stream is open. If not, throws an exception.
- *
- * @throws IOException if stream is closed
- */
- private synchronized void checkOpen() throws IOException {
- if (xceiverClient == null) {
- throw new IOException("ChunkInputStream has been closed.");
- }
- }
-
- /**
- * Prepares to read by advancing through chunks and buffers as needed until it
- * finds data to return or encounters EOF.
- *
- * @param len desired length of data to read
- * @return length of data available to read, possibly less than desired length
- */
- private synchronized int prepareRead(int len) throws IOException {
- for (;;) {
- if (chunks == null || chunks.isEmpty()) {
- // This must be an empty key.
- return EOF;
- } else if (buffers == null) {
- // The first read triggers fetching the first chunk.
- readChunkFromContainer();
- } else if (!buffers.isEmpty() &&
- buffers.get(bufferIndex).hasRemaining()) {
- // Data is available from the current buffer.
- ByteBuffer bb = buffers.get(bufferIndex);
- return len > bb.remaining() ? bb.remaining() : len;
- } else if (!buffers.isEmpty() &&
- !buffers.get(bufferIndex).hasRemaining() &&
- bufferIndex < buffers.size() - 1) {
- // There are additional buffers available.
- ++bufferIndex;
- } else if (chunkIndex < chunks.size() - 1) {
- // There are additional chunks available.
- readChunkFromContainer();
- } else {
- // All available input has been consumed.
- return EOF;
- }
- }
- }
-
- /**
- * Attempts to read the chunk at the specified offset in the chunk list. If
- * successful, then the data of the read chunk is saved so that its bytes can
- * be returned from subsequent read calls.
- *
- * @throws IOException if there is an I/O error while performing the call
- */
- private synchronized void readChunkFromContainer() throws IOException {
- // On every chunk read chunkIndex should be increased so as to read the
- // next chunk
- chunkIndex += 1;
- final ReadChunkResponseProto readChunkResponse;
- try {
- readChunkResponse = ContainerProtocolCalls.readChunk(xceiverClient,
- chunks.get(chunkIndex), key, traceID);
- } catch (IOException e) {
- throw new IOException("Unexpected OzoneException: " + e.toString(), e);
- }
- ByteString byteString = readChunkResponse.getData();
- buffers = byteString.asReadOnlyByteBufferList();
- bufferIndex = 0;
- }
-
- @Override
- public synchronized void seek(long pos) throws IOException {
- if (pos < 0 || (chunks.size() == 0 && pos > 0)
- || pos >= chunkOffset[chunks.size() - 1] + chunks.get(chunks.size() - 1)
- .getLen()) {
- throw new EOFException(
- "EOF encountered pos: " + pos + " container key: " + key);
- }
- if (chunkIndex == -1) {
- chunkIndex = Arrays.binarySearch(chunkOffset, pos);
- } else if (pos < chunkOffset[chunkIndex]) {
- chunkIndex = Arrays.binarySearch(chunkOffset, 0, chunkIndex, pos);
- } else if (pos >= chunkOffset[chunkIndex] + chunks.get(chunkIndex)
- .getLen()) {
- chunkIndex =
- Arrays.binarySearch(chunkOffset, chunkIndex + 1, chunks.size(), pos);
- }
- if (chunkIndex < 0) {
- // Binary search returns -insertionPoint - 1 if element is not present
- // in the array. insertionPoint is the point at which element would be
- // inserted in the sorted array. We need to adjust the chunkIndex
- // accordingly so that chunkIndex = insertionPoint - 1
- chunkIndex = -chunkIndex -2;
- }
- // adjust chunkIndex so that readChunkFromContainer reads the correct chunk
- chunkIndex -= 1;
- readChunkFromContainer();
- adjustBufferIndex(pos);
- }
-
- private void adjustBufferIndex(long pos) {
- long tempOffest = chunkOffset[chunkIndex];
- for (int i = 0; i < buffers.size(); i++) {
- if (pos - tempOffest >= buffers.get(i).capacity()) {
- tempOffest += buffers.get(i).capacity();
- } else {
- bufferIndex = i;
- break;
- }
- }
- buffers.get(bufferIndex).position((int) (pos - tempOffest));
- }
-
- @Override
- public synchronized long getPos() throws IOException {
- return chunkIndex == -1 ? 0 :
- chunkOffset[chunkIndex] + buffers.get(bufferIndex).position();
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
deleted file mode 100644
index 64c10da..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ChunkOutputStream.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import static org.apache.hadoop.scm.storage.ContainerProtocolCalls.putKey;
-import static org.apache.hadoop.scm.storage.ContainerProtocolCalls.writeChunk;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.nio.ByteBuffer;
-import java.util.UUID;
-
-import com.google.protobuf.ByteString;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-
-/**
- * An {@link OutputStream} used by the REST service in combination with the
- * SCMClient to write the value of a key to a sequence
- * of container chunks. Writes are buffered locally and periodically written to
- * the container as a new chunk. In order to preserve the semantics that
- * replacement of a pre-existing key is atomic, each instance of the stream has
- * an internal unique identifier. This unique identifier and a monotonically
- * increasing chunk index form a composite key that is used as the chunk name.
- * After all data is written, a putKey call creates or updates the corresponding
- * container key, and this call includes the full list of chunks that make up
- * the key data. The list of chunks is updated all at once. Therefore, a
- * concurrent reader never can see an intermediate state in which different
- * chunks of data from different versions of the key data are interleaved.
- * This class encapsulates all state management for buffering and writing
- * through to the container.
- */
-public class ChunkOutputStream extends OutputStream {
-
- private final String containerKey;
- private final String key;
- private final String traceID;
- private final KeyData.Builder containerKeyData;
- private XceiverClientManager xceiverClientManager;
- private XceiverClientSpi xceiverClient;
- private ByteBuffer buffer;
- private final String streamId;
- private int chunkIndex;
- private int chunkSize;
-
- /**
- * Creates a new ChunkOutputStream.
- *
- * @param containerKey container key
- * @param key chunk key
- * @param xceiverClientManager client manager that controls client
- * @param xceiverClient client to perform container calls
- * @param traceID container protocol call args
- * @param chunkSize chunk size
- */
- public ChunkOutputStream(String containerKey, String key,
- XceiverClientManager xceiverClientManager, XceiverClientSpi xceiverClient,
- String traceID, int chunkSize) {
- this.containerKey = containerKey;
- this.key = key;
- this.traceID = traceID;
- this.chunkSize = chunkSize;
- KeyValue keyValue = KeyValue.newBuilder()
- .setKey("TYPE").setValue("KEY").build();
- this.containerKeyData = KeyData.newBuilder()
- .setContainerName(xceiverClient.getPipeline().getContainerName())
- .setName(containerKey)
- .addMetadata(keyValue);
- this.xceiverClientManager = xceiverClientManager;
- this.xceiverClient = xceiverClient;
- this.buffer = ByteBuffer.allocate(chunkSize);
- this.streamId = UUID.randomUUID().toString();
- this.chunkIndex = 0;
- }
-
- @Override
- public synchronized void write(int b) throws IOException {
- checkOpen();
- int rollbackPosition = buffer.position();
- int rollbackLimit = buffer.limit();
- buffer.put((byte)b);
- if (buffer.position() == chunkSize) {
- flushBufferToChunk(rollbackPosition, rollbackLimit);
- }
- }
-
- @Override
- public void write(byte[] b, int off, int len) throws IOException {
- if (b == null) {
- throw new NullPointerException();
- }
- if ((off < 0) || (off > b.length) || (len < 0) ||
- ((off + len) > b.length) || ((off + len) < 0)) {
- throw new IndexOutOfBoundsException();
- }
- if (len == 0) {
- return;
- }
- checkOpen();
- while (len > 0) {
- int writeLen = Math.min(chunkSize - buffer.position(), len);
- int rollbackPosition = buffer.position();
- int rollbackLimit = buffer.limit();
- buffer.put(b, off, writeLen);
- if (buffer.position() == chunkSize) {
- flushBufferToChunk(rollbackPosition, rollbackLimit);
- }
- off += writeLen;
- len -= writeLen;
- }
- }
-
- @Override
- public synchronized void flush() throws IOException {
- checkOpen();
- if (buffer.position() > 0) {
- int rollbackPosition = buffer.position();
- int rollbackLimit = buffer.limit();
- flushBufferToChunk(rollbackPosition, rollbackLimit);
- }
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (xceiverClientManager != null && xceiverClient != null &&
- buffer != null) {
- try {
- if (buffer.position() > 0) {
- writeChunkToContainer();
- }
- putKey(xceiverClient, containerKeyData.build(), traceID);
- } catch (IOException e) {
- throw new IOException(
- "Unexpected Storage Container Exception: " + e.toString(), e);
- } finally {
- xceiverClientManager.releaseClient(xceiverClient);
- xceiverClientManager = null;
- xceiverClient = null;
- buffer = null;
- }
- }
-
- }
-
- /**
- * Checks if the stream is open. If not, throws an exception.
- *
- * @throws IOException if stream is closed
- */
- private synchronized void checkOpen() throws IOException {
- if (xceiverClient == null) {
- throw new IOException("ChunkOutputStream has been closed.");
- }
- }
-
- /**
- * Attempts to flush buffered writes by writing a new chunk to the container.
- * If successful, then clears the buffer to prepare to receive writes for a
- * new chunk.
- *
- * @param rollbackPosition position to restore in buffer if write fails
- * @param rollbackLimit limit to restore in buffer if write fails
- * @throws IOException if there is an I/O error while performing the call
- */
- private synchronized void flushBufferToChunk(int rollbackPosition,
- int rollbackLimit) throws IOException {
- boolean success = false;
- try {
- writeChunkToContainer();
- success = true;
- } finally {
- if (success) {
- buffer.clear();
- } else {
- buffer.position(rollbackPosition);
- buffer.limit(rollbackLimit);
- }
- }
- }
-
- /**
- * Writes buffered data as a new chunk to the container and saves chunk
- * information to be used later in putKey call.
- *
- * @throws IOException if there is an I/O error while performing the call
- */
- private synchronized void writeChunkToContainer() throws IOException {
- buffer.flip();
- ByteString data = ByteString.copyFrom(buffer);
- ChunkInfo chunk = ChunkInfo
- .newBuilder()
- .setChunkName(
- DigestUtils.md5Hex(key) + "_stream_"
- + streamId + "_chunk_" + ++chunkIndex)
- .setOffset(0)
- .setLen(data.size())
- .build();
- try {
- writeChunk(xceiverClient, chunk, key, data, traceID);
- } catch (IOException e) {
- throw new IOException(
- "Unexpected Storage Container Exception: " + e.toString(), e);
- }
- containerKeyData.addChunks(chunk);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
deleted file mode 100644
index 1cde67c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/ContainerProtocolCalls.java
+++ /dev/null
@@ -1,397 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ChunkInfo;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .GetKeyRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .GetKeyResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .GetSmallFileRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .GetSmallFileResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.KeyData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .PutKeyRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .PutSmallFileRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ReadChunkRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ReadChunkResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.Type;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .WriteChunkRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ReadContainerResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ReadContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.KeyValue;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-
-import java.io.IOException;
-import org.apache.hadoop.scm.XceiverClientSpi;
-
-/**
- * Implementation of all container protocol calls performed by Container
- * clients.
- */
-public final class ContainerProtocolCalls {
-
- /**
- * There is no need to instantiate this class.
- */
- private ContainerProtocolCalls() {
- }
-
- /**
- * Calls the container protocol to get a container key.
- *
- * @param xceiverClient client to perform call
- * @param containerKeyData key data to identify container
- * @param traceID container protocol call args
- * @return container protocol get key response
- * @throws IOException if there is an I/O error while performing the call
- */
- public static GetKeyResponseProto getKey(XceiverClientSpi xceiverClient,
- KeyData containerKeyData, String traceID) throws IOException {
- GetKeyRequestProto.Builder readKeyRequest = GetKeyRequestProto
- .newBuilder()
- .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
- .setKeyData(containerKeyData);
- String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request = ContainerCommandRequestProto
- .newBuilder()
- .setCmdType(Type.GetKey)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setGetKey(readKeyRequest)
- .build();
- ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
- validateContainerResponse(response);
- return response.getGetKey();
- }
-
- /**
- * Calls the container protocol to put a container key.
- *
- * @param xceiverClient client to perform call
- * @param containerKeyData key data to identify container
- * @param traceID container protocol call args
- * @throws IOException if there is an I/O error while performing the call
- */
- public static void putKey(XceiverClientSpi xceiverClient,
- KeyData containerKeyData, String traceID) throws IOException {
- PutKeyRequestProto.Builder createKeyRequest = PutKeyRequestProto
- .newBuilder()
- .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
- .setKeyData(containerKeyData);
- String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request = ContainerCommandRequestProto
- .newBuilder()
- .setCmdType(Type.PutKey)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setPutKey(createKeyRequest)
- .build();
- ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
- validateContainerResponse(response);
- }
-
- /**
- * Calls the container protocol to read a chunk.
- *
- * @param xceiverClient client to perform call
- * @param chunk information about chunk to read
- * @param key the key name
- * @param traceID container protocol call args
- * @return container protocol read chunk response
- * @throws IOException if there is an I/O error while performing the call
- */
- public static ReadChunkResponseProto readChunk(XceiverClientSpi xceiverClient,
- ChunkInfo chunk, String key, String traceID)
- throws IOException {
- ReadChunkRequestProto.Builder readChunkRequest = ReadChunkRequestProto
- .newBuilder()
- .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
- .setKeyName(key)
- .setChunkData(chunk);
- String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request = ContainerCommandRequestProto
- .newBuilder()
- .setCmdType(Type.ReadChunk)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setReadChunk(readChunkRequest)
- .build();
- ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
- validateContainerResponse(response);
- return response.getReadChunk();
- }
-
- /**
- * Calls the container protocol to write a chunk.
- *
- * @param xceiverClient client to perform call
- * @param chunk information about chunk to write
- * @param key the key name
- * @param data the data of the chunk to write
- * @param traceID container protocol call args
- * @throws IOException if there is an I/O error while performing the call
- */
- public static void writeChunk(XceiverClientSpi xceiverClient, ChunkInfo chunk,
- String key, ByteString data, String traceID)
- throws IOException {
- WriteChunkRequestProto.Builder writeChunkRequest = WriteChunkRequestProto
- .newBuilder()
- .setPipeline(xceiverClient.getPipeline().getProtobufMessage())
- .setKeyName(key)
- .setChunkData(chunk)
- .setData(data);
- String id = xceiverClient.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request = ContainerCommandRequestProto
- .newBuilder()
- .setCmdType(Type.WriteChunk)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setWriteChunk(writeChunkRequest)
- .build();
- ContainerCommandResponseProto response = xceiverClient.sendCommand(request);
- validateContainerResponse(response);
- }
-
- /**
- * Allows writing a small file using single RPC. This takes the container
- * name, key name and data to write sends all that data to the container using
- * a single RPC. This API is designed to be used for files which are smaller
- * than 1 MB.
- *
- * @param client - client that communicates with the container.
- * @param containerName - Name of the container
- * @param key - Name of the Key
- * @param data - Data to be written into the container.
- * @param traceID - Trace ID for logging purpose.
- * @throws IOException
- */
- public static void writeSmallFile(XceiverClientSpi client,
- String containerName, String key, byte[] data, String traceID)
- throws IOException {
-
- KeyData containerKeyData =
- KeyData.newBuilder().setContainerName(containerName).setName(key)
- .build();
- PutKeyRequestProto.Builder createKeyRequest =
- PutKeyRequestProto.newBuilder()
- .setPipeline(client.getPipeline().getProtobufMessage())
- .setKeyData(containerKeyData);
-
- KeyValue keyValue =
- KeyValue.newBuilder().setKey("OverWriteRequested").setValue("true")
- .build();
- ChunkInfo chunk =
- ChunkInfo.newBuilder().setChunkName(key + "_chunk").setOffset(0)
- .setLen(data.length).addMetadata(keyValue).build();
-
- PutSmallFileRequestProto putSmallFileRequest =
- PutSmallFileRequestProto.newBuilder().setChunkInfo(chunk)
- .setKey(createKeyRequest).setData(ByteString.copyFrom(data))
- .build();
-
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request =
- ContainerCommandRequestProto.newBuilder()
- .setCmdType(Type.PutSmallFile)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setPutSmallFile(putSmallFileRequest)
- .build();
- ContainerCommandResponseProto response = client.sendCommand(request);
- validateContainerResponse(response);
- }
-
- /**
- * createContainer call that creates a container on the datanode.
- * @param client - client
- * @param traceID - traceID
- * @throws IOException
- */
- public static void createContainer(XceiverClientSpi client, String traceID)
- throws IOException {
- ContainerProtos.CreateContainerRequestProto.Builder createRequest =
- ContainerProtos.CreateContainerRequestProto
- .newBuilder();
- ContainerProtos.ContainerData.Builder containerData = ContainerProtos
- .ContainerData.newBuilder();
- containerData.setName(client.getPipeline().getContainerName());
- createRequest.setPipeline(client.getPipeline().getProtobufMessage());
- createRequest.setContainerData(containerData.build());
-
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto.Builder request =
- ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.CreateContainer);
- request.setCreateContainer(createRequest);
- request.setDatanodeID(id);
- request.setTraceID(traceID);
- ContainerCommandResponseProto response = client.sendCommand(
- request.build());
- validateContainerResponse(response);
- }
-
- /**
- * Deletes a container from a pipeline.
- *
- * @param client
- * @param force whether or not to forcibly delete the container.
- * @param traceID
- * @throws IOException
- */
- public static void deleteContainer(XceiverClientSpi client,
- boolean force, String traceID) throws IOException {
- ContainerProtos.DeleteContainerRequestProto.Builder deleteRequest =
- ContainerProtos.DeleteContainerRequestProto.newBuilder();
- deleteRequest.setName(client.getPipeline().getContainerName());
- deleteRequest.setPipeline(client.getPipeline().getProtobufMessage());
- deleteRequest.setForceDelete(force);
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto.Builder request =
- ContainerCommandRequestProto.newBuilder();
- request.setCmdType(ContainerProtos.Type.DeleteContainer);
- request.setDeleteContainer(deleteRequest);
- request.setTraceID(traceID);
- request.setDatanodeID(id);
- ContainerCommandResponseProto response =
- client.sendCommand(request.build());
- validateContainerResponse(response);
- }
-
- /**
- * Close a container.
- *
- * @param client
- * @param traceID
- * @throws IOException
- */
- public static void closeContainer(XceiverClientSpi client, String traceID)
- throws IOException {
- ContainerProtos.CloseContainerRequestProto.Builder closeRequest =
- ContainerProtos.CloseContainerRequestProto.newBuilder();
- closeRequest.setPipeline(client.getPipeline().getProtobufMessage());
-
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto.Builder request =
- ContainerCommandRequestProto.newBuilder();
- request.setCmdType(Type.CloseContainer);
- request.setCloseContainer(closeRequest);
- request.setTraceID(traceID);
- request.setDatanodeID(id);
- ContainerCommandResponseProto response =
- client.sendCommand(request.build());
- validateContainerResponse(response);
- }
-
- /**
- * readContainer call that gets meta data from an existing container.
- *
- * @param client - client
- * @param traceID - trace ID
- * @throws IOException
- */
- public static ReadContainerResponseProto readContainer(
- XceiverClientSpi client, String containerName,
- String traceID) throws IOException {
- ReadContainerRequestProto.Builder readRequest =
- ReadContainerRequestProto.newBuilder();
- readRequest.setName(containerName);
- readRequest.setPipeline(client.getPipeline().getProtobufMessage());
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto.Builder request =
- ContainerCommandRequestProto.newBuilder();
- request.setCmdType(Type.ReadContainer);
- request.setReadContainer(readRequest);
- request.setDatanodeID(id);
- request.setTraceID(traceID);
- ContainerCommandResponseProto response =
- client.sendCommand(request.build());
- validateContainerResponse(response);
- return response.getReadContainer();
- }
-
- /**
- * Reads the data given the container name and key.
- *
- * @param client
- * @param containerName - name of the container
- * @param key - key
- * @param traceID - trace ID
- * @return GetSmallFileResponseProto
- * @throws IOException
- */
- public static GetSmallFileResponseProto readSmallFile(XceiverClientSpi client,
- String containerName, String key, String traceID) throws IOException {
- KeyData containerKeyData = KeyData
- .newBuilder()
- .setContainerName(containerName)
- .setName(key).build();
-
- GetKeyRequestProto.Builder getKey = GetKeyRequestProto
- .newBuilder()
- .setPipeline(client.getPipeline().getProtobufMessage())
- .setKeyData(containerKeyData);
- ContainerProtos.GetSmallFileRequestProto getSmallFileRequest =
- GetSmallFileRequestProto
- .newBuilder().setKey(getKey)
- .build();
- String id = client.getPipeline().getLeader().getDatanodeUuid();
- ContainerCommandRequestProto request = ContainerCommandRequestProto
- .newBuilder()
- .setCmdType(Type.GetSmallFile)
- .setTraceID(traceID)
- .setDatanodeID(id)
- .setGetSmallFile(getSmallFileRequest)
- .build();
- ContainerCommandResponseProto response = client.sendCommand(request);
- validateContainerResponse(response);
- return response.getGetSmallFile();
- }
-
- /**
- * Validates a response from a container protocol call. Any non-successful
- * return code is mapped to a corresponding exception and thrown.
- *
- * @param response container protocol call response
- * @throws IOException if the container protocol call failed
- */
- private static void validateContainerResponse(
- ContainerCommandResponseProto response
- ) throws StorageContainerException {
- if (response.getResult() == ContainerProtos.Result.SUCCESS) {
- return;
- }
- throw new StorageContainerException(
- response.getMessage(), response.getResult());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
deleted file mode 100644
index aa89af0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/storage/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.storage;
-
-/**
- * This package contains StorageContainerManager classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
deleted file mode 100644
index de159ab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/RatisHelper.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ratis;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftGroupId;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.util.SizeInBytes;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Collections;
-import java.util.Collection;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.stream.Collectors;
-
-/**
- * Ratis helper methods.
- */
-public interface RatisHelper {
- Logger LOG = LoggerFactory.getLogger(RatisHelper.class);
-
- static String toRaftPeerIdString(DatanodeID id) {
- return id.getIpAddr() + "_" + id.getRatisPort();
- }
-
- static String toRaftPeerAddressString(DatanodeID id) {
- return id.getIpAddr() + ":" + id.getRatisPort();
- }
-
- static RaftPeerId toRaftPeerId(DatanodeID id) {
- return RaftPeerId.valueOf(toRaftPeerIdString(id));
- }
-
- static RaftPeer toRaftPeer(DatanodeID id) {
- return new RaftPeer(toRaftPeerId(id), toRaftPeerAddressString(id));
- }
-
- static List<RaftPeer> toRaftPeers(Pipeline pipeline) {
- return toRaftPeers(pipeline.getMachines());
- }
-
- static <E extends DatanodeID> List<RaftPeer> toRaftPeers(List<E> datanodes) {
- return datanodes.stream().map(RatisHelper::toRaftPeer)
- .collect(Collectors.toList());
- }
-
- /* TODO: use a dummy id for all groups for the moment.
- * It should be changed to a unique id for each group.
- */
- RaftGroupId DUMMY_GROUP_ID =
- RaftGroupId.valueOf(ByteString.copyFromUtf8("AOzoneRatisGroup"));
-
- RaftGroup EMPTY_GROUP = new RaftGroup(DUMMY_GROUP_ID,
- Collections.emptyList());
-
- static RaftGroup emptyRaftGroup() {
- return EMPTY_GROUP;
- }
-
- static RaftGroup newRaftGroup(List<DatanodeID> datanodes) {
- final List<RaftPeer> newPeers = datanodes.stream()
- .map(RatisHelper::toRaftPeer)
- .collect(Collectors.toList());
- return RatisHelper.newRaftGroup(newPeers);
- }
-
- static RaftGroup newRaftGroup(Collection<RaftPeer> peers) {
- return peers.isEmpty()? emptyRaftGroup()
- : new RaftGroup(DUMMY_GROUP_ID, peers);
- }
-
- static RaftGroup newRaftGroup(Pipeline pipeline) {
- return newRaftGroup(toRaftPeers(pipeline));
- }
-
- static RaftClient newRaftClient(RpcType rpcType, Pipeline pipeline) {
- return newRaftClient(rpcType, toRaftPeerId(pipeline.getLeader()),
- newRaftGroup(pipeline));
- }
-
- static RaftClient newRaftClient(RpcType rpcType, RaftPeer leader) {
- return newRaftClient(rpcType, leader.getId(),
- newRaftGroup(new ArrayList<>(Arrays.asList(leader))));
- }
-
- static RaftClient newRaftClient(
- RpcType rpcType, RaftPeerId leader, RaftGroup group) {
- LOG.trace("newRaftClient: {}, leader={}, group={}", rpcType, leader, group);
- final RaftProperties properties = new RaftProperties();
- RaftConfigKeys.Rpc.setType(properties, rpcType);
- GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE));
-
- return RaftClient.newBuilder()
- .setRaftGroup(group)
- .setLeaderId(leader)
- .setProperties(properties)
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
deleted file mode 100644
index c13c20c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis;
-
-/**
- * This package contains classes related to Apache Ratis.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
deleted file mode 100644
index 29242ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/ShadedProtoUtil.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.shaded.com.google.protobuf;
-
-/** Utilities for the shaded protobuf in Ratis. */
-public interface ShadedProtoUtil {
- /**
- * @param bytes
- * @return the wrapped shaded {@link ByteString} (no coping).
- */
- static ByteString asShadedByteString(byte[] bytes) {
- return ByteString.wrap(bytes);
- }
-
- /**
- * @param shaded
- * @return a {@link com.google.protobuf.ByteString} (require coping).
- */
- static com.google.protobuf.ByteString asByteString(ByteString shaded) {
- return com.google.protobuf.ByteString.copyFrom(
- shaded.asReadOnlyByteBuffer());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
deleted file mode 100644
index 032dd96..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/ratis/shaded/com/google/protobuf/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.ratis.shaded.com.google.protobuf;
-
-/**
- * This package contains classes related to the shaded protobuf in Apache Ratis.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
deleted file mode 100644
index 4e79827..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/DatanodeContainerProtocol.proto
+++ /dev/null
@@ -1,415 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and Unstable.
- * Please see http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/InterfaceClassification.html
- * for what changes are allowed for a *Unstable* .proto interface.
- */
-
-// This file contains protocol buffers that are used to transfer data
-// to and from the datanode.
-option java_package = "org.apache.hadoop.hdfs.ozone.protocol.proto";
-option java_outer_classname = "ContainerProtos";
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-import "hdfs.proto";
-import "Ozone.proto";
-
-/**
- * Commands that are used to manipulate the state of containers on a datanode.
- *
- * These commands allow us to work against the datanode - from
- * StorageContainer Manager as well as clients.
- *
- * 1. CreateContainer - This call is usually made by Storage Container
- * manager, when we need to create a new container on a given datanode.
- *
- * 2. ReadContainer - Allows end user to stat a container. For example
- * this allows us to return the metadata of a container.
- *
- * 3. UpdateContainer - Updates a container metadata.
-
- * 4. DeleteContainer - This call is made to delete a container.
- *
- * 5. ListContainer - Returns the list of containers on this
- * datanode. This will be used by tests and tools.
- *
- * 6. PutKey - Given a valid container, creates a key.
- *
- * 7. GetKey - Allows user to read the metadata of a Key.
- *
- * 8. DeleteKey - Deletes a given key.
- *
- * 9. ListKey - Returns a list of keys that are present inside
- * a given container.
- *
- * 10. ReadChunk - Allows us to read a chunk.
- *
- * 11. DeleteChunk - Delete an unused chunk.
- *
- * 12. WriteChunk - Allows us to write a chunk
- *
- * 13. ListChunk - Given a Container/Key returns the list of Chunks.
- *
- * 14. CompactChunk - Re-writes a chunk based on Offsets.
- *
- * 15. PutSmallFile - A single RPC that combines both putKey and WriteChunk.
- *
- * 16. GetSmallFile - A single RPC that combines both getKey and ReadChunk.
- *
- * 17. CloseContainer - Closes an open container and makes it immutable.
- *
- * 18. CopyContainer - Copies a container from a remote machine.
- */
-
-enum Type {
- CreateContainer = 1;
- ReadContainer = 2;
- UpdateContainer = 3;
- DeleteContainer = 4;
- ListContainer = 5;
-
- PutKey = 6;
- GetKey = 7;
- DeleteKey = 8;
- ListKey = 9;
-
- ReadChunk = 10;
- DeleteChunk = 11;
- WriteChunk = 12;
- ListChunk = 13;
- CompactChunk = 14;
-
- /** Combines Key and Chunk Operation into Single RPC. */
- PutSmallFile = 15;
- GetSmallFile = 16;
- CloseContainer = 17;
-
-}
-
-
-enum Result {
- SUCCESS = 1;
- UNSUPPORTED_REQUEST = 2;
- MALFORMED_REQUEST = 3;
- CONTAINER_INTERNAL_ERROR = 4;
- INVALID_CONFIG = 5;
- INVALID_FILE_HASH_FOUND = 6;
- CONTAINER_EXISTS = 7;
- NO_SUCH_ALGORITHM = 8;
- CONTAINER_NOT_FOUND = 9;
- IO_EXCEPTION = 10;
- UNABLE_TO_READ_METADATA_DB = 11;
- NO_SUCH_KEY = 12;
- OVERWRITE_FLAG_REQUIRED = 13;
- UNABLE_TO_FIND_DATA_DIR = 14;
- INVALID_WRITE_SIZE = 15;
- CHECKSUM_MISMATCH = 16;
- UNABLE_TO_FIND_CHUNK = 17;
- PROTOC_DECODING_ERROR = 18;
- INVALID_ARGUMENT = 19;
- PUT_SMALL_FILE_ERROR = 20;
- GET_SMALL_FILE_ERROR = 21;
- CLOSED_CONTAINER_IO = 22;
- ERROR_CONTAINER_NOT_EMPTY = 23;
- ERROR_IN_COMPACT_DB = 24;
- UNCLOSED_CONTAINER_IO = 25;
- DELETE_ON_OPEN_CONTAINER = 26;
- CLOSED_CONTAINER_RETRY = 27;
-}
-
-message ContainerCommandRequestProto {
- required Type cmdType = 1; // Type of the command
-
- // A string that identifies this command, we generate Trace ID in Ozone
- // frontend and this allows us to trace that command all over ozone.
- optional string traceID = 2;
-
- // One of the following command is available when the corresponding
- // cmdType is set. At the protocol level we allow only
- // one command in each packet.
- // TODO : Upgrade to Protobuf 2.6 or later.
- optional CreateContainerRequestProto createContainer = 3;
- optional ReadContainerRequestProto readContainer = 4;
- optional UpdateContainerRequestProto updateContainer = 5;
- optional DeleteContainerRequestProto deleteContainer = 6;
- optional ListContainerRequestProto listContainer = 7;
-
- optional PutKeyRequestProto putKey = 8;
- optional GetKeyRequestProto getKey = 9;
- optional DeleteKeyRequestProto deleteKey = 10;
- optional ListKeyRequestProto listKey = 11;
-
- optional ReadChunkRequestProto readChunk = 12;
- optional WriteChunkRequestProto writeChunk = 13;
- optional DeleteChunkRequestProto deleteChunk = 14;
- optional ListChunkRequestProto listChunk = 15;
-
- optional PutSmallFileRequestProto putSmallFile = 16;
- optional GetSmallFileRequestProto getSmallFile = 17;
- optional CloseContainerRequestProto closeContainer = 18;
- required string datanodeID = 19;
-}
-
-message ContainerCommandResponseProto {
- required Type cmdType = 1;
- optional string traceID = 2;
-
- optional CreateContainerResponseProto createContainer = 3;
- optional ReadContainerResponseProto readContainer = 4;
- optional UpdateContainerResponseProto updateContainer = 5;
- optional DeleteContainerResponseProto deleteContainer = 6;
- optional ListContainerResponseProto listContainer = 7;
-
- optional PutKeyResponseProto putKey = 8;
- optional GetKeyResponseProto getKey = 9;
- optional DeleteKeyResponseProto deleteKey = 10;
- optional ListKeyResponseProto listKey = 11;
-
- optional WriteChunkResponseProto writeChunk = 12;
- optional ReadChunkResponseProto readChunk = 13;
- optional DeleteChunkResponseProto deleteChunk = 14;
- optional ListChunkResponseProto listChunk = 15;
-
- required Result result = 17;
- optional string message = 18;
-
- optional PutSmallFileResponseProto putSmallFile = 19;
- optional GetSmallFileResponseProto getSmallFile = 20;
- optional CloseContainerResponseProto closeContainer = 21;
-
-}
-
-message ContainerData {
- required string name = 1;
- repeated KeyValue metadata = 2;
- optional string dbPath = 3;
- optional string containerPath = 4;
- optional string hash = 6;
- optional int64 bytesUsed = 7;
- optional int64 size = 8;
- optional int64 keyCount = 9;
- //TODO: change required after we switch container ID from string to long
- optional int64 containerID = 10;
- optional LifeCycleState state = 11 [default = OPEN];
-}
-
-message ContainerMeta {
- required string fileName = 1;
- required string hash = 2;
-}
-
-// Container Messages.
-message CreateContainerRequestProto {
- required Pipeline pipeline = 1;
- required ContainerData containerData = 2;
-}
-
-message CreateContainerResponseProto {
-}
-
-message ReadContainerRequestProto {
- required Pipeline pipeline = 1;
- required string name = 2;
-}
-
-message ReadContainerResponseProto {
- optional ContainerData containerData = 2;
-}
-
-message UpdateContainerRequestProto {
- required Pipeline pipeline = 1;
- required ContainerData containerData = 2;
- optional bool forceUpdate = 3 [default = false];
-}
-
-message UpdateContainerResponseProto {
-}
-
-message DeleteContainerRequestProto {
- required Pipeline pipeline = 1;
- required string name = 2;
- optional bool forceDelete = 3 [default = false];
-}
-
-message DeleteContainerResponseProto {
-}
-
-message ListContainerRequestProto {
- required Pipeline pipeline = 1;
- optional string prefix = 2;
- required uint32 count = 3; // Max Results to return
- optional string prevKey = 4; // if this is not set query from start.
-}
-
-message ListContainerResponseProto {
- repeated ContainerData containerData = 1;
-}
-
-message CloseContainerRequestProto {
- required Pipeline pipeline = 1;
-}
-
-message CloseContainerResponseProto {
- optional Pipeline pipeline = 1;
- optional string hash = 2;
-}
-
-message KeyData {
- required string containerName = 1;
- required string name = 2;
- optional int64 flags = 3; // for future use.
- repeated KeyValue metadata = 4;
- repeated ChunkInfo chunks = 5;
-}
-
-// Key Messages.
-message PutKeyRequestProto {
- required Pipeline pipeline = 1;
- required KeyData keyData = 2;
-}
-
-message PutKeyResponseProto {
-}
-
-message GetKeyRequestProto {
- required Pipeline pipeline = 1;
- required KeyData keyData = 2;
-}
-
-message GetKeyResponseProto {
- required KeyData keyData = 1;
-}
-
-
-message DeleteKeyRequestProto {
- required Pipeline pipeline = 1;
- required string name = 2;
-}
-
-message DeleteKeyResponseProto {
-}
-
-message ListKeyRequestProto {
- required Pipeline pipeline = 1;
- optional string prefix = 2; // if specified returns keys that match prefix.
- required string prevKey = 3;
- required uint32 count = 4;
-
-}
-
-message ListKeyResponseProto {
- repeated KeyData keyData = 1;
-}
-
-// Chunk Operations
-
-message ChunkInfo {
- required string chunkName = 1;
- required uint64 offset = 2;
- required uint64 len = 3;
- optional string checksum = 4;
- repeated KeyValue metadata = 5;
-}
-
-enum Stage {
- WRITE_DATA = 1;
- COMMIT_DATA = 2;
- COMBINED = 3;
-}
-
-message WriteChunkRequestProto {
- required Pipeline pipeline = 1;
- required string keyName = 2;
- required ChunkInfo chunkData = 3;
- optional bytes data = 4;
- optional Stage stage = 5 [default = COMBINED];
-}
-
-message WriteChunkResponseProto {
-}
-
-message ReadChunkRequestProto {
- required Pipeline pipeline = 1;
- required string keyName = 2;
- required ChunkInfo chunkData = 3;
-}
-
-message ReadChunkResponseProto {
- required Pipeline pipeline = 1;
- required ChunkInfo chunkData = 2;
- required bytes data = 3;
-}
-
-message DeleteChunkRequestProto {
- required Pipeline pipeline = 1;
- required string keyName = 2;
- required ChunkInfo chunkData = 3;
-}
-
-message DeleteChunkResponseProto {
-}
-
-message ListChunkRequestProto {
- required Pipeline pipeline = 1;
- required string keyName = 2;
- required string prevChunkName = 3;
- required uint32 count = 4;
-}
-
-message ListChunkResponseProto {
- repeated ChunkInfo chunkData = 1;
-}
-
-/** For small file access combines write chunk and putKey into a single
-RPC */
-
-message PutSmallFileRequestProto {
- required PutKeyRequestProto key = 1;
- required ChunkInfo chunkInfo = 2;
- required bytes data = 3;
-}
-
-
-message PutSmallFileResponseProto {
-
-}
-
-message GetSmallFileRequestProto {
- required GetKeyRequestProto key = 1;
-}
-
-message GetSmallFileResponseProto {
- required ReadChunkResponseProto data = 1;
-}
-
-message CopyContainerRequestProto {
- required string containerName = 1;
- required uint64 readOffset = 2;
- optional uint64 len = 3;
-}
-
-message CopyContainerResponseProto {
- required string archiveName = 1;
- required uint64 readOffset = 2;
- required uint64 len = 3;
- required bool eof = 4;
- repeated bytes data = 5;
- optional int64 checksum = 6;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
deleted file mode 100644
index cdc1df5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/KeySpaceManagerProtocol.proto
+++ /dev/null
@@ -1,457 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "KeySpaceManagerProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-
-/**
-This is file contains the protocol to communicate with
-Ozone key space manager. Ozone KSM manages the namespace for ozone.
-This is similar to Namenode for Ozone.
-*/
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-enum Status {
- OK = 1;
- VOLUME_NOT_UNIQUE = 2;
- VOLUME_NOT_FOUND = 3;
- VOLUME_NOT_EMPTY = 4;
- VOLUME_ALREADY_EXISTS = 5;
- USER_NOT_FOUND = 6;
- USER_TOO_MANY_VOLUMES = 7;
- BUCKET_NOT_FOUND = 8;
- BUCKET_NOT_EMPTY = 9;
- BUCKET_ALREADY_EXISTS = 10;
- KEY_ALREADY_EXISTS = 11;
- KEY_NOT_FOUND = 12;
- ACCESS_DENIED = 13;
- INTERNAL_ERROR = 14;
-}
-
-
-message VolumeInfo {
- required string adminName = 1;
- required string ownerName = 2;
- required string volume = 3;
- optional uint64 quotaInBytes = 4;
- repeated KeyValue metadata = 5;
- repeated OzoneAclInfo volumeAcls = 6;
- required uint64 creationTime = 7;
-}
-
-/**
- Creates a volume
-*/
-message CreateVolumeRequest {
- required VolumeInfo volumeInfo = 1;
-}
-
-message CreateVolumeResponse {
-
- required Status status = 1;
-}
-
-message VolumeList {
- repeated string volumeNames = 1;
-}
-
-/**
- Changes the Volume Properties -- like ownership and quota for a volume.
-*/
-message SetVolumePropertyRequest {
- required string volumeName = 1;
- optional string ownerName = 2;
- optional uint64 quotaInBytes = 3;
-}
-
-message SetVolumePropertyResponse {
- required Status status = 1;
-}
-
-/**
- * Checks if the user has specified permissions for the volume
- */
-message CheckVolumeAccessRequest {
- required string volumeName = 1;
- required OzoneAclInfo userAcl = 2;
-}
-
-message CheckVolumeAccessResponse {
-
- required Status status = 1;
-}
-
-
-/**
- Returns information about a volume.
-*/
-
-message InfoVolumeRequest {
- required string volumeName = 1;
-}
-
-message InfoVolumeResponse {
- required Status status = 1;
- optional VolumeInfo volumeInfo = 2;
-
-}
-
-/**
- Deletes an existing volume.
-*/
-message DeleteVolumeRequest {
- required string volumeName = 1;
-}
-
-message DeleteVolumeResponse {
- required Status status = 1;
-}
-
-
-/**
- List Volumes -- List all volumes in the cluster or by user.
-*/
-
-message ListVolumeRequest {
- enum Scope {
- USER_VOLUMES = 1; // User volumes -- called by user
- VOLUMES_BY_USER = 2; // User volumes - called by Admin
- VOLUMES_BY_CLUSTER = 3; // All volumes in the cluster
- }
- required Scope scope = 1;
- optional string userName = 2;
- optional string prefix = 3;
- optional string prevKey = 4;
- optional uint32 maxKeys = 5;
-}
-
-message ListVolumeResponse {
- required Status status = 1;
- repeated VolumeInfo volumeInfo = 2;
-}
-
-message BucketInfo {
- required string volumeName = 1;
- required string bucketName = 2;
- repeated OzoneAclInfo acls = 3;
- required bool isVersionEnabled = 4 [default = false];
- required StorageTypeProto storageType = 5 [default = DISK];
- required uint64 creationTime = 6;
-}
-
-message BucketArgs {
- required string volumeName = 1;
- required string bucketName = 2;
- repeated OzoneAclInfo addAcls = 3;
- repeated OzoneAclInfo removeAcls = 4;
- optional bool isVersionEnabled = 5;
- optional StorageTypeProto storageType = 6;
-}
-
-message OzoneAclInfo {
- enum OzoneAclType {
- USER = 1;
- GROUP = 2;
- WORLD = 3;
- }
- enum OzoneAclRights {
- READ = 1;
- WRITE = 2;
- READ_WRITE = 3;
- }
- required OzoneAclType type = 1;
- required string name = 2;
- required OzoneAclRights rights = 3;
-}
-
-message CreateBucketRequest {
- required BucketInfo bucketInfo = 1;
-}
-
-message CreateBucketResponse {
- required Status status = 1;
-}
-
-message InfoBucketRequest {
- required string volumeName = 1;
- required string bucketName = 2;
-}
-
-message InfoBucketResponse {
- required Status status = 1;
- optional BucketInfo bucketInfo = 2;
-}
-
-message ListBucketsRequest {
- required string volumeName = 1;
- optional string startKey = 2;
- optional string prefix = 3;
- optional int32 count = 4;
-}
-
-message ListBucketsResponse {
- required Status status = 1;
- repeated BucketInfo bucketInfo = 2;
-}
-
-message KeyArgs {
- required string volumeName = 1;
- required string bucketName = 2;
- required string keyName = 3;
- optional uint64 dataSize = 4;
- optional hadoop.hdfs.ozone.ReplicationType type = 5;
- optional hadoop.hdfs.ozone.ReplicationFactor factor = 6;
-}
-
-message KeyLocation {
- required string blockID = 1;
- required string containerName = 2;
- required bool shouldCreateContainer = 3;
- required uint64 offset = 4;
- required uint64 length = 5;
- // indicated at which version this block gets created.
- optional uint64 createVersion = 6;
-}
-
-message KeyLocationList {
- optional uint64 version = 1;
- repeated KeyLocation keyLocations = 2;
-}
-
-message KeyInfo {
- required string volumeName = 1;
- required string bucketName = 2;
- required string keyName = 3;
- required uint64 dataSize = 4;
- repeated KeyLocationList keyLocationList = 5;
- required uint64 creationTime = 6;
- required uint64 modificationTime = 7;
- optional uint64 latestVersion = 8;
-}
-
-message LocateKeyRequest {
- required KeyArgs keyArgs = 1;
-}
-
-message LocateKeyResponse {
- required Status status = 1;
- optional KeyInfo keyInfo = 2;
- // clients' followup request may carry this ID for stateful operations (similar
- // to a cookie).
- optional uint32 ID = 3;
- // TODO : allow specifiying a particular version to read.
- optional uint64 openVersion = 4;
-}
-
-message SetBucketPropertyRequest {
- required BucketArgs bucketArgs = 1;
-}
-
-message SetBucketPropertyResponse {
- required Status status = 1;
-}
-
-message DeleteBucketRequest {
- required string volumeName = 1;
- required string bucketName = 2;
-}
-
-message DeleteBucketResponse {
- required Status status = 1;
-}
-
-message ListKeysRequest {
- required string volumeName = 1;
- required string bucketName = 2;
- optional string startKey = 3;
- optional string prefix = 4;
- optional int32 count = 5;
-}
-
-message ListKeysResponse {
- required Status status = 1;
- repeated KeyInfo keyInfo = 2;
-}
-
-message AllocateBlockRequest {
- required KeyArgs keyArgs = 1;
- required uint32 clientID = 2;
-}
-
-message AllocateBlockResponse {
- required Status status = 1;
- required KeyLocation keyLocation = 2;
-}
-
-message CommitKeyRequest {
- required KeyArgs keyArgs = 1;
- required uint32 clientID = 2;
-}
-
-message CommitKeyResponse {
- required Status status = 1;
-}
-
-message ServiceListRequest {
-}
-
-message ServiceListResponse {
- required Status status = 1;
- repeated ServiceInfo serviceInfo = 2;
-}
-
-message ServicePort {
- enum Type {
- RPC = 1;
- HTTP = 2;
- HTTPS = 3;
- RATIS = 4;
- };
- required Type type = 1;
- required uint32 value = 2;
-}
-
-message ServiceInfo {
- required NodeType nodeType = 1;
- required string hostname = 2;
- repeated ServicePort servicePorts = 3;
-}
-
-/**
- The KSM service that takes care of Ozone namespace.
-*/
-service KeySpaceManagerService {
-
- /**
- Creates a Volume.
- */
- rpc createVolume(CreateVolumeRequest)
- returns(CreateVolumeResponse);
-
- /**
- Allows modificiation of volume properties.
- */
- rpc setVolumeProperty(SetVolumePropertyRequest)
- returns (SetVolumePropertyResponse);
-
- /**
- Checks if the specified volume is accesible by the specified user.
- */
- rpc checkVolumeAccess(CheckVolumeAccessRequest)
- returns (CheckVolumeAccessResponse);
-
- /**
- Gets Volume information.
- */
- rpc infoVolume(InfoVolumeRequest)
- returns(InfoVolumeResponse);
- /**
- Deletes a volume if it is empty.
- */
- rpc deleteVolume(DeleteVolumeRequest)
- returns (DeleteVolumeResponse);
-
- /**
- Lists Volumes
- */
- rpc listVolumes(ListVolumeRequest)
- returns (ListVolumeResponse);
-
- /**
- Creates a Bucket.
- */
- rpc createBucket(CreateBucketRequest)
- returns(CreateBucketResponse);
-
- /**
- Get Bucket information.
- */
- rpc infoBucket(InfoBucketRequest)
- returns(InfoBucketResponse);
-
- /**
- Sets bucket properties.
- */
- rpc setBucketProperty(SetBucketPropertyRequest)
- returns(SetBucketPropertyResponse);
-
- /**
- Get key.
- */
- rpc createKey(LocateKeyRequest)
- returns(LocateKeyResponse);
-
- /**
- Look up for an existing key.
- */
- rpc lookupKey(LocateKeyRequest)
- returns(LocateKeyResponse);
-
- /**
- Delete an existing key.
- */
- rpc deleteKey(LocateKeyRequest)
- returns(LocateKeyResponse);
-
- /**
- Deletes a bucket from volume if it is empty.
- */
- rpc deleteBucket(DeleteBucketRequest)
- returns (DeleteBucketResponse);
-
- /**
- List Buckets.
- */
- rpc listBuckets(ListBucketsRequest)
- returns(ListBucketsResponse);
-
- /**
- List Keys.
- */
- rpc listKeys(ListKeysRequest)
- returns(ListKeysResponse);
-
- /**
- Commit a key.
- */
- rpc commitKey(CommitKeyRequest)
- returns(CommitKeyResponse);
-
- /**
- Allocate a new block for a key.
- */
- rpc allocateBlock(AllocateBlockRequest)
- returns(AllocateBlockResponse);
-
- /**
- Returns list of Ozone services with its configuration details.
- */
- rpc getServiceList(ServiceListRequest)
- returns(ServiceListResponse);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
deleted file mode 100644
index 3acbcd6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManagerImpl.java
+++ /dev/null
@@ -1,512 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BatchOperation;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone
- .OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE_IN_MB;
-import org.apache.hadoop.ozone.protocol
- .proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol
- .proto.OzoneProtos.ReplicationFactor;
-
-
-/**
- * Implementation of keyManager.
- */
-public class KeyManagerImpl implements KeyManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(KeyManagerImpl.class);
-
- /**
- * A SCM block client, used to talk to SCM to allocate block during putKey.
- */
- private final ScmBlockLocationProtocol scmBlockClient;
- private final KSMMetadataManager metadataManager;
- private final long scmBlockSize;
- private final boolean useRatis;
- private final BackgroundService keyDeletingService;
- private final BackgroundService openKeyCleanupService;
-
- private final long preallocateMax;
- private final Random random;
- private final String ksmId;
-
- public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
- KSMMetadataManager metadataManager, OzoneConfiguration conf,
- String ksmId) {
- this.scmBlockClient = scmBlockClient;
- this.metadataManager = metadataManager;
- this.scmBlockSize = conf.getLong(OZONE_SCM_BLOCK_SIZE_IN_MB,
- OZONE_SCM_BLOCK_SIZE_DEFAULT) * OzoneConsts.MB;
- this.useRatis = conf.getBoolean(DFS_CONTAINER_RATIS_ENABLED_KEY,
- DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
- long blockDeleteInterval = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- long serviceTimeout = conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.preallocateMax = conf.getLong(
- OZONE_KEY_PREALLOCATION_MAXSIZE,
- OZONE_KEY_PREALLOCATION_MAXSIZE_DEFAULT);
- keyDeletingService = new KeyDeletingService(
- scmBlockClient, this, blockDeleteInterval, serviceTimeout, conf);
- int openkeyCheckInterval = conf.getInt(
- OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS,
- OZONE_OPEN_KEY_CLEANUP_SERVICE_INTERVAL_SECONDS_DEFAULT);
- openKeyCleanupService = new OpenKeyCleanupService(
- scmBlockClient, this, openkeyCheckInterval, serviceTimeout);
- random = new Random();
- this.ksmId = ksmId;
- }
-
- @VisibleForTesting
- public BackgroundService getOpenKeyCleanupService() {
- return openKeyCleanupService;
- }
-
- @Override
- public void start() {
- keyDeletingService.start();
- openKeyCleanupService.start();
- }
-
- @Override
- public void stop() throws IOException {
- keyDeletingService.shutdown();
- openKeyCleanupService.shutdown();
- }
-
- private void validateBucket(String volumeName, String bucketName)
- throws IOException {
- byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-
- //Check if the volume exists
- if(metadataManager.get(volumeKey) == null) {
- LOG.error("volume not found: {}", volumeName);
- throw new KSMException("Volume not found",
- KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
- //Check if bucket already exists
- if(metadataManager.get(bucketKey) == null) {
- LOG.error("bucket not found: {}/{} ", volumeName, bucketName);
- throw new KSMException("Bucket not found",
- KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
- }
-
- @Override
- public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
- throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
- ReplicationFactor factor = args.getFactor();
- ReplicationType type = args.getType();
-
- // If user does not specify a replication strategy or
- // replication factor, KSM will use defaults.
- if(factor == null) {
- factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
- }
-
- if(type == null) {
- type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
- }
-
- try {
- validateBucket(volumeName, bucketName);
- String objectKey = metadataManager.getKeyWithDBPrefix(
- volumeName, bucketName, keyName);
- byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID);
- byte[] keyData = metadataManager.get(openKey);
- if (keyData == null) {
- LOG.error("Allocate block for a key not in open status in meta store " +
- objectKey + " with ID " + clientID);
- throw new KSMException("Open Key not found",
- KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
- }
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(scmBlockSize, type, factor, ksmId);
- KsmKeyInfo keyInfo =
- KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(keyData));
- KsmKeyLocationInfo info = new KsmKeyLocationInfo.Builder()
- .setContainerName(allocatedBlock.getPipeline().getContainerName())
- .setBlockID(allocatedBlock.getKey())
- .setShouldCreateContainer(allocatedBlock.getCreateContainer())
- .setLength(scmBlockSize)
- .setOffset(0)
- .build();
- // current version not committed, so new blocks coming now are added to
- // the same version
- keyInfo.appendNewBlocks(Collections.singletonList(info));
- keyInfo.updateModifcationTime();
- metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray());
- return info;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
- ReplicationFactor factor = args.getFactor();
- ReplicationType type = args.getType();
-
- // If user does not specify a replication strategy or
- // replication factor, KSM will use defaults.
- if(factor == null) {
- factor = useRatis ? ReplicationFactor.THREE: ReplicationFactor.ONE;
- }
-
- if(type == null) {
- type = useRatis ? ReplicationType.RATIS : ReplicationType.STAND_ALONE;
- }
-
- try {
- validateBucket(volumeName, bucketName);
- long requestedSize = Math.min(preallocateMax, args.getDataSize());
- List<KsmKeyLocationInfo> locations = new ArrayList<>();
- String objectKey = metadataManager.getKeyWithDBPrefix(
- volumeName, bucketName, keyName);
- // requested size is not required but more like a optimization:
- // SCM looks at the requested, if it 0, no block will be allocated at
- // the point, if client needs more blocks, client can always call
- // allocateBlock. But if requested size is not 0, KSM will preallocate
- // some blocks and piggyback to client, to save RPC calls.
- while (requestedSize > 0) {
- long allocateSize = Math.min(scmBlockSize, requestedSize);
- AllocatedBlock allocatedBlock =
- scmBlockClient.allocateBlock(allocateSize, type, factor, ksmId);
- KsmKeyLocationInfo subKeyInfo = new KsmKeyLocationInfo.Builder()
- .setContainerName(allocatedBlock.getPipeline().getContainerName())
- .setBlockID(allocatedBlock.getKey())
- .setShouldCreateContainer(allocatedBlock.getCreateContainer())
- .setLength(allocateSize)
- .setOffset(0)
- .build();
- locations.add(subKeyInfo);
- requestedSize -= allocateSize;
- }
- // NOTE size of a key is not a hard limit on anything, it is a value that
- // client should expect, in terms of current size of key. If client sets a
- // value, then this value is used, otherwise, we allocate a single block
- // which is the current size, if read by the client.
- long size = args.getDataSize() >= 0 ? args.getDataSize() : scmBlockSize;
- byte[] keyKey = metadataManager.getDBKeyBytes(
- volumeName, bucketName, keyName);
- byte[] value = metadataManager.get(keyKey);
- KsmKeyInfo keyInfo;
- long openVersion;
- if (value != null) {
- // the key already exist, the new blocks will be added as new version
- keyInfo = KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
- // when locations.size = 0, the new version will have identical blocks
- // as its previous version
- openVersion = keyInfo.addNewVersion(locations);
- keyInfo.setDataSize(size + keyInfo.getDataSize());
- } else {
- // the key does not exist, create a new object, the new blocks are the
- // version 0
- long currentTime = Time.now();
- keyInfo = new KsmKeyInfo.Builder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setKsmKeyLocationInfos(Collections.singletonList(
- new KsmKeyLocationInfoGroup(0, locations)))
- .setCreationTime(currentTime)
- .setModificationTime(currentTime)
- .setDataSize(size)
- .build();
- openVersion = 0;
- }
- // Generate a random ID which is not already in meta db.
- int id = -1;
- // in general this should finish in a couple times at most. putting some
- // arbitrary large number here to avoid dead loop.
- for (int j = 0; j < 10000; j++) {
- id = random.nextInt();
- byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, id);
- if (metadataManager.get(openKey) == null) {
- metadataManager.put(openKey, keyInfo.getProtobuf().toByteArray());
- break;
- }
- }
- if (id == -1) {
- throw new IOException("Failed to find a usable id for " + objectKey);
- }
- LOG.debug("Key {} allocated in volume {} bucket {}",
- keyName, volumeName, bucketName);
- return new OpenKeySession(id, keyInfo, openVersion);
- } catch (KSMException e) {
- throw e;
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Key open failed for volume:{} bucket:{} key:{}",
- volumeName, bucketName, keyName, ex);
- }
- throw new KSMException(ex.getMessage(),
- KSMException.ResultCodes.FAILED_KEY_ALLOCATION);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public void commitKey(KsmKeyArgs args, int clientID) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
- try {
- validateBucket(volumeName, bucketName);
- String objectKey = metadataManager.getKeyWithDBPrefix(
- volumeName, bucketName, keyName);
- byte[] objectKeyBytes = metadataManager.getDBKeyBytes(volumeName,
- bucketName, keyName);
- byte[] openKey = metadataManager.getOpenKeyNameBytes(objectKey, clientID);
- byte[] openKeyData = metadataManager.get(openKey);
- if (openKeyData == null) {
- throw new KSMException("Commit a key without corresponding entry " +
- DFSUtil.bytes2String(openKey), ResultCodes.FAILED_KEY_NOT_FOUND);
- }
- KsmKeyInfo keyInfo =
- KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(openKeyData));
- keyInfo.setDataSize(args.getDataSize());
- keyInfo.setModificationTime(Time.now());
- BatchOperation batch = new BatchOperation();
- batch.delete(openKey);
- batch.put(objectKeyBytes, keyInfo.getProtobuf().toByteArray());
- metadataManager.writeBatch(batch);
- } catch (KSMException e) {
- throw e;
- } catch (IOException ex) {
- LOG.error("Key commit failed for volume:{} bucket:{} key:{}",
- volumeName, bucketName, keyName, ex);
- throw new KSMException(ex.getMessage(),
- KSMException.ResultCodes.FAILED_KEY_ALLOCATION);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
- try {
- byte[] keyKey = metadataManager.getDBKeyBytes(
- volumeName, bucketName, keyName);
- byte[] value = metadataManager.get(keyKey);
- if (value == null) {
- LOG.debug("volume:{} bucket:{} Key:{} not found",
- volumeName, bucketName, keyName);
- throw new KSMException("Key not found",
- KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
- }
- return KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(value));
- } catch (DBException ex) {
- LOG.error("Get key failed for volume:{} bucket:{} key:{}",
- volumeName, bucketName, keyName, ex);
- throw new KSMException(ex.getMessage(),
- KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public void deleteKey(KsmKeyArgs args) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
- try {
- byte[] objectKey = metadataManager.getDBKeyBytes(
- volumeName, bucketName, keyName);
- byte[] objectValue = metadataManager.get(objectKey);
- if (objectValue == null) {
- throw new KSMException("Key not found",
- KSMException.ResultCodes.FAILED_KEY_NOT_FOUND);
- }
- byte[] deletingKey = metadataManager.getDeletedKeyName(objectKey);
- BatchOperation batch = new BatchOperation();
- batch.put(deletingKey, objectValue);
- batch.delete(objectKey);
- metadataManager.writeBatch(batch);
- } catch (DBException ex) {
- LOG.error(String.format("Delete key failed for volume:%s "
- + "bucket:%s key:%s", volumeName, bucketName, keyName), ex);
- throw new KSMException(ex.getMessage(), ex,
- ResultCodes.FAILED_KEY_DELETION);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
-
- metadataManager.readLock().lock();
- try {
- return metadataManager.listKeys(volumeName, bucketName,
- startKey, keyPrefix, maxKeys);
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- @Override
- public List<BlockGroup> getPendingDeletionKeys(final int count)
- throws IOException {
- metadataManager.readLock().lock();
- try {
- return metadataManager.getPendingDeletionKeys(count);
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- @Override
- public void deletePendingDeletionKey(String objectKeyName)
- throws IOException{
- Preconditions.checkNotNull(objectKeyName);
- if (!objectKeyName.startsWith(OzoneConsts.DELETING_KEY_PREFIX)) {
- throw new IllegalArgumentException("Invalid key name,"
- + " the name should be the key name with deleting prefix");
- }
-
- // Simply removes the entry from KSM DB.
- metadataManager.writeLock().lock();
- try {
- byte[] pendingDelKey = DFSUtil.string2Bytes(objectKeyName);
- byte[] delKeyValue = metadataManager.get(pendingDelKey);
- if (delKeyValue == null) {
- throw new IOException("Failed to delete key " + objectKeyName
- + " because it is not found in DB");
- }
- metadataManager.delete(pendingDelKey);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- @Override
- public List<BlockGroup> getExpiredOpenKeys() throws IOException {
- metadataManager.readLock().lock();
- try {
- return metadataManager.getExpiredOpenKeys();
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- @Override
- public void deleteExpiredOpenKey(String objectKeyName) throws IOException {
- Preconditions.checkNotNull(objectKeyName);
- if (!objectKeyName.startsWith(OzoneConsts.OPEN_KEY_PREFIX)) {
- throw new IllegalArgumentException("Invalid key name,"
- + " the name should be the key name with open key prefix");
- }
-
- // Simply removes the entry from KSM DB.
- metadataManager.writeLock().lock();
- try {
- byte[] openKey = DFSUtil.string2Bytes(objectKeyName);
- byte[] delKeyValue = metadataManager.get(openKey);
- if (delKeyValue == null) {
- throw new IOException("Failed to delete key " + objectKeyName
- + " because it is not found in DB");
- }
- metadataManager.delete(openKey);
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
deleted file mode 100644
index 6139292..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManager.java
+++ /dev/null
@@ -1,902 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.protobuf.BlockingService;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.jmx.ServiceRuntimeInfoImpl;
-import org.apache.hadoop.ozone.common.Storage.StorageState;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos
- .ServicePort;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB
- .KeySpaceManagerProtocolServerSideTranslatorPB;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB
- .ScmBlockLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.KeySpaceManagerService
- .newReflectiveBlockingService;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
- .NodeState.HEALTHY;
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-/**
- * Ozone Keyspace manager is the metadata manager of ozone.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
-public class KeySpaceManager extends ServiceRuntimeInfoImpl
- implements KeySpaceManagerProtocol, KSMMXBean {
- private static final Logger LOG =
- LoggerFactory.getLogger(KeySpaceManager.class);
-
- private static final String USAGE =
- "Usage: \n hdfs ksm [genericOptions] " + "[ "
- + StartupOption.CREATEOBJECTSTORE.getName() + " ]\n " + "hdfs ksm [ "
- + StartupOption.HELP.getName() + " ]\n";
-
- /** Startup options. */
- public enum StartupOption {
- CREATEOBJECTSTORE("-createObjectStore"),
- HELP("-help"),
- REGULAR("-regular");
-
- private final String name;
-
- StartupOption(String arg) {
- this.name = arg;
- }
-
- public String getName() {
- return name;
- }
-
- public static StartupOption parse(String value) {
- for (StartupOption option : StartupOption.values()) {
- if (option.name.equalsIgnoreCase(value)) {
- return option;
- }
- }
- return null;
- }
- }
-
- private final OzoneConfiguration configuration;
- private final RPC.Server ksmRpcServer;
- private final InetSocketAddress ksmRpcAddress;
- private final KSMMetadataManager metadataManager;
- private final VolumeManager volumeManager;
- private final BucketManager bucketManager;
- private final KeyManager keyManager;
- private final KSMMetrics metrics;
- private final KeySpaceManagerHttpServer httpServer;
- private final KSMStorage ksmStorage;
- private final ScmBlockLocationProtocol scmBlockClient;
- private final StorageContainerLocationProtocol scmContainerClient;
- private ObjectName ksmInfoBeanName;
-
- private KeySpaceManager(OzoneConfiguration conf) throws IOException {
- Preconditions.checkNotNull(conf);
- configuration = conf;
- ksmStorage = new KSMStorage(conf);
- scmBlockClient = getScmBlockClient(configuration);
- scmContainerClient = getScmContainerClient(configuration);
- if (ksmStorage.getState() != StorageState.INITIALIZED) {
- throw new KSMException("KSM not initialized.",
- ResultCodes.KSM_NOT_INITIALIZED);
- }
-
- // verifies that the SCM info in the KSM Version file is correct.
- ScmInfo scmInfo = scmBlockClient.getScmInfo();
- if (!(scmInfo.getClusterId().equals(ksmStorage.getClusterID()) && scmInfo
- .getScmId().equals(ksmStorage.getScmId()))) {
- throw new KSMException("SCM version info mismatch.",
- ResultCodes.SCM_VERSION_MISMATCH_ERROR);
- }
- final int handlerCount = conf.getInt(OZONE_KSM_HANDLER_COUNT_KEY,
- OZONE_KSM_HANDLER_COUNT_DEFAULT);
-
- RPC.setProtocolEngine(configuration, KeySpaceManagerProtocolPB.class,
- ProtobufRpcEngine.class);
-
- BlockingService ksmService = newReflectiveBlockingService(
- new KeySpaceManagerProtocolServerSideTranslatorPB(this));
- final InetSocketAddress ksmNodeRpcAddr = OzoneClientUtils.
- getKsmAddress(configuration);
- ksmRpcServer = startRpcServer(configuration, ksmNodeRpcAddr,
- KeySpaceManagerProtocolPB.class, ksmService,
- handlerCount);
- ksmRpcAddress = OzoneClientUtils.updateRPCListenAddress(configuration,
- OZONE_KSM_ADDRESS_KEY, ksmNodeRpcAddr, ksmRpcServer);
- metadataManager = new KSMMetadataManagerImpl(configuration);
- volumeManager = new VolumeManagerImpl(metadataManager, configuration);
- bucketManager = new BucketManagerImpl(metadataManager);
- metrics = KSMMetrics.create();
- keyManager =
- new KeyManagerImpl(scmBlockClient, metadataManager, configuration,
- ksmStorage.getKsmId());
- httpServer = new KeySpaceManagerHttpServer(configuration, this);
- }
-
- /**
- * Create a scm block client, used by putKey() and getKey().
- *
- * @return {@link ScmBlockLocationProtocol}
- * @throws IOException
- */
- private static ScmBlockLocationProtocol getScmBlockClient(
- OzoneConfiguration conf) throws IOException {
- RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- long scmVersion =
- RPC.getProtocolVersion(ScmBlockLocationProtocolPB.class);
- InetSocketAddress scmBlockAddress =
- OzoneClientUtils.getScmAddressForBlockClients(conf);
- ScmBlockLocationProtocolClientSideTranslatorPB scmBlockLocationClient =
- new ScmBlockLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
- scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
- return scmBlockLocationClient;
- }
-
- /**
- * Returns a scm container client.
- *
- * @return {@link StorageContainerLocationProtocol}
- * @throws IOException
- */
- private static StorageContainerLocationProtocol getScmContainerClient(
- OzoneConfiguration conf) throws IOException {
- RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- long scmVersion =
- RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
- InetSocketAddress scmAddr = OzoneClientUtils.getScmAddressForClients(
- conf);
- StorageContainerLocationProtocolClientSideTranslatorPB scmContainerClient =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddr, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
- return scmContainerClient;
- }
-
- @VisibleForTesting
- public KeyManager getKeyManager() {
- return keyManager;
- }
-
- @VisibleForTesting
- public ScmInfo getScmInfo() throws IOException {
- return scmBlockClient.getScmInfo();
- }
-
- @VisibleForTesting
- public KSMStorage getKsmStorage() {
- return ksmStorage;
- }
- /**
- * Starts an RPC server, if configured.
- *
- * @param conf configuration
- * @param addr configured address of RPC server
- * @param protocol RPC protocol provided by RPC server
- * @param instance RPC protocol implementation instance
- * @param handlerCount RPC server handler count
- *
- * @return RPC server
- * @throws IOException if there is an I/O error while creating RPC server
- */
- private static RPC.Server startRpcServer(OzoneConfiguration conf,
- InetSocketAddress addr, Class<?> protocol, BlockingService instance,
- int handlerCount) throws IOException {
- RPC.Server rpcServer = new RPC.Builder(conf)
- .setProtocol(protocol)
- .setInstance(instance)
- .setBindAddress(addr.getHostString())
- .setPort(addr.getPort())
- .setNumHandlers(handlerCount)
- .setVerbose(false)
- .setSecretManager(null)
- .build();
-
- DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
- return rpcServer;
- }
-
- /**
- * Get metadata manager.
- * @return metadata manager.
- */
- public KSMMetadataManager getMetadataManager() {
- return metadataManager;
- }
-
- public KSMMetrics getMetrics() {
- return metrics;
- }
-
- /**
- * Main entry point for starting KeySpaceManager.
- *
- * @param argv arguments
- * @throws IOException if startup fails due to I/O error
- */
- public static void main(String[] argv) throws IOException {
- if (DFSUtil.parseHelpArgument(argv, USAGE, System.out, true)) {
- System.exit(0);
- }
- try {
- OzoneConfiguration conf = new OzoneConfiguration();
- GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
- if (!hParser.isParseSuccessful()) {
- System.err.println("USAGE: " + USAGE + " \n");
- hParser.printGenericCommandUsage(System.err);
- System.exit(1);
- }
- StringUtils.startupShutdownMessage(KeySpaceManager.class, argv, LOG);
- KeySpaceManager ksm = createKSM(hParser.getRemainingArgs(), conf);
- if (ksm != null) {
- ksm.start();
- ksm.join();
- }
- } catch (Throwable t) {
- LOG.error("Failed to start the KeyspaceManager.", t);
- terminate(1, t);
- }
- }
-
- private static void printUsage(PrintStream out) {
- out.println(USAGE + "\n");
- }
-
- /**
- * Constructs KSM instance based on command line arguments.
- * @param argv Command line arguments
- * @param conf OzoneConfiguration
- * @return KSM instance
- * @throws IOException in case KSM instance creation fails.
- */
-
- public static KeySpaceManager createKSM(String[] argv,
- OzoneConfiguration conf) throws IOException {
- if (!DFSUtil.isOzoneEnabled(conf)) {
- System.err.println("KSM cannot be started in secure mode or when " +
- OZONE_ENABLED + " is set to false");
- System.exit(1);
- }
- StartupOption startOpt = parseArguments(argv);
- if (startOpt == null) {
- printUsage(System.err);
- terminate(1);
- return null;
- }
- switch (startOpt) {
- case CREATEOBJECTSTORE:
- terminate(ksmInit(conf) ? 0 : 1);
- return null;
- case HELP:
- printUsage(System.err);
- terminate(0);
- return null;
- default:
- return new KeySpaceManager(conf);
- }
- }
-
- /**
- * Initializes the KSM instance.
- * @param conf OzoneConfiguration
- * @return true if KSM initialization succeeds , false otherwise
- * @throws IOException in case ozone metadata directory path is not accessible
- */
-
- private static boolean ksmInit(OzoneConfiguration conf) throws IOException {
- KSMStorage ksmStorage = new KSMStorage(conf);
- StorageState state = ksmStorage.getState();
- if (state != StorageState.INITIALIZED) {
- try {
- ScmBlockLocationProtocol scmBlockClient = getScmBlockClient(conf);
- ScmInfo scmInfo = scmBlockClient.getScmInfo();
- String clusterId = scmInfo.getClusterId();
- String scmId = scmInfo.getScmId();
- if (clusterId == null || clusterId.isEmpty()) {
- throw new IOException("Invalid Cluster ID");
- }
- if (scmId == null || scmId.isEmpty()) {
- throw new IOException("Invalid SCM ID");
- }
- ksmStorage.setClusterId(clusterId);
- ksmStorage.setScmId(scmId);
- ksmStorage.initialize();
- System.out.println(
- "KSM initialization succeeded.Current cluster id for sd="
- + ksmStorage.getStorageDir() + ";cid=" + ksmStorage
- .getClusterID());
- return true;
- } catch (IOException ioe) {
- LOG.error("Could not initialize KSM version file", ioe);
- return false;
- }
- } else {
- System.out.println(
- "KSM already initialized.Reusing existing cluster id for sd="
- + ksmStorage.getStorageDir() + ";cid=" + ksmStorage
- .getClusterID());
- return true;
- }
- }
-
- /**
- * Parses the command line options for KSM initialization.
- * @param args command line arguments
- * @return StartupOption if options are valid, null otherwise
- */
- private static StartupOption parseArguments(String[] args) {
- if (args == null || args.length == 0) {
- return StartupOption.REGULAR;
- } else if (args.length == 1) {
- return StartupOption.parse(args[0]);
- }
- return null;
- }
-
- /**
- * Builds a message for logging startup information about an RPC server.
- *
- * @param description RPC server description
- * @param addr RPC server listening address
- * @return server startup message
- */
- private static String buildRpcServerStartMessage(String description,
- InetSocketAddress addr) {
- return addr != null ? String.format("%s is listening at %s",
- description, addr.toString()) :
- String.format("%s not started", description);
- }
-
- /**
- * Start service.
- */
- public void start() throws IOException {
- LOG.info(buildRpcServerStartMessage("KeyspaceManager RPC server",
- ksmRpcAddress));
- DefaultMetricsSystem.initialize("KeySpaceManager");
- metadataManager.start();
- keyManager.start();
- ksmRpcServer.start();
- httpServer.start();
- registerMXBean();
- setStartTime();
- }
-
- /**
- * Stop service.
- */
- public void stop() {
- try {
- metadataManager.stop();
- ksmRpcServer.stop();
- keyManager.stop();
- httpServer.stop();
- metrics.unRegister();
- unregisterMXBean();
- } catch (Exception e) {
- LOG.error("Key Space Manager stop failed.", e);
- }
- }
-
- /**
- * Wait until service has completed shutdown.
- */
- public void join() {
- try {
- ksmRpcServer.join();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.info("Interrupted during KeyspaceManager join.", e);
- }
- }
-
- /**
- * Creates a volume.
- *
- * @param args - Arguments to create Volume.
- * @throws IOException
- */
- @Override
- public void createVolume(KsmVolumeArgs args) throws IOException {
- try {
- metrics.incNumVolumeCreates();
- volumeManager.createVolume(args);
- } catch (Exception ex) {
- metrics.incNumVolumeCreateFails();
- throw ex;
- }
- }
-
- /**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
- */
- @Override
- public void setOwner(String volume, String owner) throws IOException {
- try {
- metrics.incNumVolumeUpdates();
- volumeManager.setOwner(volume, owner);
- } catch (Exception ex) {
- metrics.incNumVolumeUpdateFails();
- throw ex;
- }
- }
-
- /**
- * Changes the Quota on a volume.
- *
- * @param volume - Name of the volume.
- * @param quota - Quota in bytes.
- * @throws IOException
- */
- @Override
- public void setQuota(String volume, long quota) throws IOException {
- try {
- metrics.incNumVolumeUpdates();
- volumeManager.setQuota(volume, quota);
- } catch (Exception ex) {
- metrics.incNumVolumeUpdateFails();
- throw ex;
- }
- }
-
- /**
- * Checks if the specified user can access this volume.
- *
- * @param volume - volume
- * @param userAcl - user acls which needs to be checked for access
- * @return true if the user has required access for the volume,
- * false otherwise
- * @throws IOException
- */
- @Override
- public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException {
- try {
- metrics.incNumVolumeCheckAccesses();
- return volumeManager.checkVolumeAccess(volume, userAcl);
- } catch (Exception ex) {
- metrics.incNumVolumeCheckAccessFails();
- throw ex;
- }
- }
-
- /**
- * Gets the volume information.
- *
- * @param volume - Volume name.
- * @return VolumeArgs or exception is thrown.
- * @throws IOException
- */
- @Override
- public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
- try {
- metrics.incNumVolumeInfos();
- return volumeManager.getVolumeInfo(volume);
- } catch (Exception ex) {
- metrics.incNumVolumeInfoFails();
- throw ex;
- }
- }
-
- /**
- * Deletes an existing empty volume.
- *
- * @param volume - Name of the volume.
- * @throws IOException
- */
- @Override
- public void deleteVolume(String volume) throws IOException {
- try {
- metrics.incNumVolumeDeletes();
- volumeManager.deleteVolume(volume);
- } catch (Exception ex) {
- metrics.incNumVolumeDeleteFails();
- throw ex;
- }
- }
-
- /**
- * Lists volume owned by a specific user.
- *
- * @param userName - user name
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the
- * prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- @Override
- public List<KsmVolumeArgs> listVolumeByUser(String userName, String prefix,
- String prevKey, int maxKeys) throws IOException {
- try {
- metrics.incNumVolumeLists();
- return volumeManager.listVolumes(userName, prefix, prevKey, maxKeys);
- } catch (Exception ex) {
- metrics.incNumVolumeListFails();
- throw ex;
- }
- }
-
- /**
- * Lists volume all volumes in the cluster.
- *
- * @param prefix - Filter prefix -- Return only entries that match this.
- * @param prevKey - Previous key -- List starts from the next from the
- * prevkey
- * @param maxKeys - Max number of keys to return.
- * @return List of Volumes.
- * @throws IOException
- */
- @Override
- public List<KsmVolumeArgs> listAllVolumes(String prefix, String prevKey, int
- maxKeys) throws IOException {
- try {
- metrics.incNumVolumeLists();
- return volumeManager.listVolumes(null, prefix, prevKey, maxKeys);
- } catch (Exception ex) {
- metrics.incNumVolumeListFails();
- throw ex;
- }
- }
-
- /**
- * Creates a bucket.
- *
- * @param bucketInfo - BucketInfo to create bucket.
- * @throws IOException
- */
- @Override
- public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
- try {
- metrics.incNumBucketCreates();
- bucketManager.createBucket(bucketInfo);
- } catch (Exception ex) {
- metrics.incNumBucketCreateFails();
- throw ex;
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KsmBucketInfo> listBuckets(String volumeName,
- String startKey, String prefix, int maxNumOfBuckets)
- throws IOException {
- try {
- metrics.incNumBucketLists();
- return bucketManager.listBuckets(volumeName,
- startKey, prefix, maxNumOfBuckets);
- } catch (IOException ex) {
- metrics.incNumBucketListFails();
- throw ex;
- }
- }
-
- /**
- * Gets the bucket information.
- *
- * @param volume - Volume name.
- * @param bucket - Bucket name.
- * @return KsmBucketInfo or exception is thrown.
- * @throws IOException
- */
- @Override
- public KsmBucketInfo getBucketInfo(String volume, String bucket)
- throws IOException {
- try {
- metrics.incNumBucketInfos();
- return bucketManager.getBucketInfo(volume, bucket);
- } catch (Exception ex) {
- metrics.incNumBucketInfoFails();
- throw ex;
- }
- }
-
- /**
- * Allocate a key.
- *
- * @param args - attributes of the key.
- * @return KsmKeyInfo - the info about the allocated key.
- * @throws IOException
- */
- @Override
- public OpenKeySession openKey(KsmKeyArgs args) throws IOException {
- try {
- metrics.incNumKeyAllocates();
- return keyManager.openKey(args);
- } catch (Exception ex) {
- metrics.incNumKeyAllocateFails();
- throw ex;
- }
- }
-
- @Override
- public void commitKey(KsmKeyArgs args, int clientID)
- throws IOException {
- try {
- metrics.incNumKeyCommits();
- keyManager.commitKey(args, clientID);
- } catch (Exception ex) {
- metrics.incNumKeyCommitFails();
- throw ex;
- }
- }
-
- @Override
- public KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
- throws IOException {
- try {
- metrics.incNumBlockAllocateCalls();
- return keyManager.allocateBlock(args, clientID);
- } catch (Exception ex) {
- metrics.incNumBlockAllocateCallFails();
- throw ex;
- }
- }
-
- /**
- * Lookup a key.
- *
- * @param args - attributes of the key.
- * @return KsmKeyInfo - the info about the requested key.
- * @throws IOException
- */
- @Override
- public KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException {
- try {
- metrics.incNumKeyLookups();
- return keyManager.lookupKey(args);
- } catch (Exception ex) {
- metrics.incNumKeyLookupFails();
- throw ex;
- }
- }
-
- /**
- * Deletes an existing key.
- *
- * @param args - attributes of the key.
- * @throws IOException
- */
- @Override
- public void deleteKey(KsmKeyArgs args) throws IOException {
- try {
- metrics.incNumKeyDeletes();
- keyManager.deleteKey(args);
- } catch (Exception ex) {
- metrics.incNumKeyDeleteFails();
- throw ex;
- }
- }
-
- @Override
- public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
- try {
- metrics.incNumKeyLists();
- return keyManager.listKeys(volumeName, bucketName,
- startKey, keyPrefix, maxKeys);
- } catch (IOException ex) {
- metrics.incNumKeyListFails();
- throw ex;
- }
- }
-
- /**
- * Sets bucket property from args.
- * @param args - BucketArgs.
- * @throws IOException
- */
- @Override
- public void setBucketProperty(KsmBucketArgs args)
- throws IOException {
- try {
- metrics.incNumBucketUpdates();
- bucketManager.setBucketProperty(args);
- } catch (Exception ex) {
- metrics.incNumBucketUpdateFails();
- throw ex;
- }
- }
-
-
- /**
- * Deletes an existing empty bucket from volume.
- * @param volume - Name of the volume.
- * @param bucket - Name of the bucket.
- * @throws IOException
- */
- public void deleteBucket(String volume, String bucket) throws IOException {
- try {
- metrics.incNumBucketDeletes();
- bucketManager.deleteBucket(volume, bucket);
- } catch (Exception ex) {
- metrics.incNumBucketDeleteFails();
- throw ex;
- }
- }
-
- private void registerMXBean() {
- Map<String, String> jmxProperties = new HashMap<String, String>();
- jmxProperties.put("component", "ServerRuntime");
- this.ksmInfoBeanName =
- MBeans.register("KeySpaceManager",
- "KeySpaceManagerInfo",
- jmxProperties,
- this);
- }
-
- private void unregisterMXBean() {
- if (this.ksmInfoBeanName != null) {
- MBeans.unregister(this.ksmInfoBeanName);
- this.ksmInfoBeanName = null;
- }
- }
-
- @Override
- public String getRpcPort() {
- return "" + ksmRpcAddress.getPort();
- }
-
- @VisibleForTesting
- public KeySpaceManagerHttpServer getHttpServer() {
- return httpServer;
- }
-
- @Override
- public List<ServiceInfo> getServiceList() throws IOException {
- // When we implement multi-home this call has to be handled properly.
- List<ServiceInfo> services = new ArrayList<>();
- ServiceInfo.Builder ksmServiceInfoBuilder = ServiceInfo.newBuilder()
- .setNodeType(OzoneProtos.NodeType.KSM)
- .setHostname(ksmRpcAddress.getHostName())
- .addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.RPC)
- .setValue(ksmRpcAddress.getPort())
- .build());
- if (httpServer.getHttpAddress() != null) {
- ksmServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.HTTP)
- .setValue(httpServer.getHttpAddress().getPort())
- .build());
- }
- if (httpServer.getHttpsAddress() != null) {
- ksmServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.HTTPS)
- .setValue(httpServer.getHttpsAddress().getPort())
- .build());
- }
- services.add(ksmServiceInfoBuilder.build());
-
- // For client we have to return SCM with container protocol port,
- // not block protocol.
- InetSocketAddress scmAddr = OzoneClientUtils.getScmAddressForClients(
- configuration);
- ServiceInfo.Builder scmServiceInfoBuilder = ServiceInfo.newBuilder()
- .setNodeType(OzoneProtos.NodeType.SCM)
- .setHostname(scmAddr.getHostName())
- .addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.RPC)
- .setValue(scmAddr.getPort()).build());
- services.add(scmServiceInfoBuilder.build());
-
- List<OzoneProtos.Node> nodes = scmContainerClient.queryNode(
- EnumSet.of(HEALTHY), OzoneProtos.QueryScope.CLUSTER, "")
- .getNodesList();
-
- for (OzoneProtos.Node node : nodes) {
- HdfsProtos.DatanodeIDProto datanode = node.getNodeID();
-
- ServiceInfo.Builder dnServiceInfoBuilder = ServiceInfo.newBuilder()
- .setNodeType(OzoneProtos.NodeType.DATANODE)
- .setHostname(datanode.getHostName());
-
- dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.HTTP)
- .setValue(datanode.getInfoPort())
- .build());
-
- if (datanode.hasInfoSecurePort() && datanode.getInfoSecurePort() > 0) {
- dnServiceInfoBuilder.addServicePort(ServicePort.newBuilder()
- .setType(ServicePort.Type.HTTPS)
- .setValue(datanode.getInfoSecurePort())
- .build());
- }
- services.add(dnServiceInfoBuilder.build());
- }
-
- metrics.incNumGetServiceLists();
- // For now there is no exception that can can happen in this call,
- // so failure metrics is not handled. In future if there is any need to
- // handle exception in this method, we need to incorporate
- // metrics.incNumGetServiceListFails()
- return services;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
deleted file mode 100644
index b229c89..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeySpaceManagerHttpServer.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.web.OzoneHttpServer;
-
-import java.io.IOException;
-
-/**
- * HttpServer wrapper for the KeySpaceManager.
- */
-public class KeySpaceManagerHttpServer extends OzoneHttpServer {
-
- public KeySpaceManagerHttpServer(Configuration conf, KeySpaceManager ksm)
- throws IOException {
- super(conf, "ksm");
- addServlet("serviceList", "/serviceList", ServiceListJSONServlet.class);
- getWebAppContext().setAttribute(OzoneConsts.KSM_CONTEXT_ATTRIBUTE, ksm);
- }
-
- @Override protected String getHttpAddressKey() {
- return KSMConfigKeys.OZONE_KSM_HTTP_ADDRESS_KEY;
- }
-
- @Override protected String getHttpBindHostKey() {
- return KSMConfigKeys.OZONE_KSM_HTTP_BIND_HOST_KEY;
- }
-
- @Override protected String getHttpsAddressKey() {
- return KSMConfigKeys.OZONE_KSM_HTTPS_ADDRESS_KEY;
- }
-
- @Override protected String getHttpsBindHostKey() {
- return KSMConfigKeys.OZONE_KSM_HTTPS_BIND_HOST_KEY;
- }
-
- @Override protected String getBindHostDefault() {
- return KSMConfigKeys.OZONE_KSM_HTTP_BIND_HOST_DEFAULT;
- }
-
- @Override protected int getHttpBindPortDefault() {
- return KSMConfigKeys.OZONE_KSM_HTTP_BIND_PORT_DEFAULT;
- }
-
- @Override protected int getHttpsBindPortDefault() {
- return KSMConfigKeys.OZONE_KSM_HTTPS_BIND_PORT_DEFAULT;
- }
-
- @Override protected String getKeytabFile() {
- return KSMConfigKeys.OZONE_KSM_KEYTAB_FILE;
- }
-
- @Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
- }
-
- @Override protected String getEnabledKey() {
- return KSMConfigKeys.OZONE_KSM_HTTP_ENABLED_KEY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
deleted file mode 100644
index 7f60bf8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/OpenKeyCleanupService.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * This is the background service to delete hanging open keys.
- * Scan the metadata of ksm periodically to get
- * the keys with prefix "#open#" and ask scm to
- * delete metadata accordingly, if scm returns
- * success for keys, then clean up those keys.
- */
-public class OpenKeyCleanupService extends BackgroundService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(OpenKeyCleanupService.class);
-
- private final static int OPEN_KEY_DELETING_CORE_POOL_SIZE = 2;
-
- private final KeyManager keyManager;
- private final ScmBlockLocationProtocol scmClient;
-
- public OpenKeyCleanupService(ScmBlockLocationProtocol scmClient,
- KeyManager keyManager, int serviceInterval,
- long serviceTimeout) {
- super("OpenKeyCleanupService", serviceInterval, TimeUnit.SECONDS,
- OPEN_KEY_DELETING_CORE_POOL_SIZE, serviceTimeout);
- this.keyManager = keyManager;
- this.scmClient = scmClient;
- }
-
- @Override
- public BackgroundTaskQueue getTasks() {
- BackgroundTaskQueue queue = new BackgroundTaskQueue();
- queue.add(new OpenKeyDeletingTask());
- return queue;
- }
-
- private class OpenKeyDeletingTask
- implements BackgroundTask<BackgroundTaskResult> {
-
- @Override
- public int getPriority() {
- return 0;
- }
-
- @Override
- public BackgroundTaskResult call() throws Exception {
- try {
- List<BlockGroup> keyBlocksList = keyManager.getExpiredOpenKeys();
- if (keyBlocksList.size() > 0) {
- int toDeleteSize = keyBlocksList.size();
- LOG.debug("Found {} to-delete open keys in KSM", toDeleteSize);
- List<DeleteBlockGroupResult> results =
- scmClient.deleteKeyBlocks(keyBlocksList);
- int deletedSize = 0;
- for (DeleteBlockGroupResult result : results) {
- if (result.isSuccess()) {
- try {
- keyManager.deleteExpiredOpenKey(result.getObjectKey());
- LOG.debug("Key {} deleted from KSM DB", result.getObjectKey());
- deletedSize += 1;
- } catch (IOException e) {
- LOG.warn("Failed to delete hanging-open key {}",
- result.getObjectKey(), e);
- }
- } else {
- LOG.warn("Deleting open Key {} failed because some of the blocks"
- + " were failed to delete, failed blocks: {}",
- result.getObjectKey(),
- String.join(",", result.getFailedBlocks()));
- }
- }
- LOG.info("Found {} expired open key entries, successfully " +
- "cleaned up {} entries", toDeleteSize, deletedSize);
- return results::size;
- } else {
- LOG.debug("No hanging open key fond in KSM");
- }
- } catch (IOException e) {
- LOG.error("Unable to get hanging open keys, retry in"
- + " next interval", e);
- }
- return BackgroundTaskResult.EmptyTaskResult.newResult();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
deleted file mode 100644
index 34a80ce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/ServiceListJSONServlet.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.SerializationFeature;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.io.PrintWriter;
-
-
-/**
- * Provides REST access to Ozone Service List.
- * <p>
- * This servlet generally will be placed under the /serviceList URL of
- * KeySpaceManager HttpServer.
- *
- * The return format is of JSON and in the form
- * <p>
- * <code><pre>
- * {
- * "services" : [
- * {
- * "NodeType":"KSM",
- * "Hostname" "$hostname",
- * "ports" : {
- * "$PortType" : "$port",
- * ...
- * }
- * }
- * ]
- * }
- * </pre></code>
- * <p>
- *
- */
-public class ServiceListJSONServlet extends HttpServlet {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ServiceListJSONServlet.class);
- private static final long serialVersionUID = 1L;
-
- private KeySpaceManager ksm;
-
- public void init() throws ServletException {
- this.ksm = (KeySpaceManager) getServletContext()
- .getAttribute(OzoneConsts.KSM_CONTEXT_ATTRIBUTE);
- }
-
- /**
- * Process a GET request for the specified resource.
- *
- * @param request
- * The servlet request we are processing
- * @param response
- * The servlet response we are creating
- */
- @Override
- public void doGet(HttpServletRequest request, HttpServletResponse response) {
- try {
- ObjectMapper objectMapper = new ObjectMapper();
- objectMapper.enable(SerializationFeature.INDENT_OUTPUT);
- response.setContentType("application/json; charset=utf8");
- PrintWriter writer = response.getWriter();
- try {
- writer.write(objectMapper.writeValueAsString(ksm.getServiceList()));
- } finally {
- if (writer != null) {
- writer.close();
- }
- }
- } catch (IOException e) {
- LOG.error(
- "Caught an exception while processing ServiceList request", e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
deleted file mode 100644
index 6ac78d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManager.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KSM volume manager interface.
- */
-public interface VolumeManager {
-
- /**
- * Create a new volume.
- * @param args - Volume args to create a volume
- */
- void createVolume(KsmVolumeArgs args) throws IOException;
-
- /**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
- */
- void setOwner(String volume, String owner) throws IOException;
-
- /**
- * Changes the Quota on a volume.
- *
- * @param volume - Name of the volume.
- * @param quota - Quota in bytes.
- * @throws IOException
- */
- void setQuota(String volume, long quota) throws IOException;
-
- /**
- * Gets the volume information.
- * @param volume - Volume name.
- * @return VolumeArgs or exception is thrown.
- * @throws IOException
- */
- KsmVolumeArgs getVolumeInfo(String volume) throws IOException;
-
- /**
- * Deletes an existing empty volume.
- *
- * @param volume - Name of the volume.
- * @throws IOException
- */
- void deleteVolume(String volume) throws IOException;
-
- /**
- * Checks if the specified user with a role can access this volume.
- *
- * @param volume - volume
- * @param userAcl - user acl which needs to be checked for access
- * @return true if the user has access for the volume, false otherwise
- * @throws IOException
- */
- boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException;
-
- /**
- * Returns a list of volumes owned by a given user; if user is null,
- * returns all volumes.
- *
- * @param userName
- * volume owner
- * @param prefix
- * the volume prefix used to filter the listing result.
- * @param startKey
- * the start volume name determines where to start listing from,
- * this key is excluded from the result.
- * @param maxKeys
- * the maximum number of volumes to return.
- * @return a list of {@link KsmVolumeArgs}
- * @throws IOException
- */
- List<KsmVolumeArgs> listVolumes(String userName, String prefix,
- String startKey, int maxKeys) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
deleted file mode 100644
index e9b094f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/VolumeManagerImpl.java
+++ /dev/null
@@ -1,391 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_USER_MAX_VOLUME_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_USER_MAX_VOLUME;
-import static org.apache.hadoop.ozone.ksm.exceptions
- .KSMException.ResultCodes;
-
-/**
- * KSM volume management code.
- */
-public class VolumeManagerImpl implements VolumeManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(VolumeManagerImpl.class);
-
- private final KSMMetadataManager metadataManager;
- private final int maxUserVolumeCount;
-
- /**
- * Constructor.
- * @param conf - Ozone configuration.
- * @throws IOException
- */
- public VolumeManagerImpl(KSMMetadataManager metadataManager,
- OzoneConfiguration conf) throws IOException {
- this.metadataManager = metadataManager;
- this.maxUserVolumeCount = conf.getInt(OZONE_KSM_USER_MAX_VOLUME,
- OZONE_KSM_USER_MAX_VOLUME_DEFAULT);
- }
-
- // Helpers to add and delete volume from user list
- private void addVolumeToOwnerList(String volume, String owner,
- BatchOperation batchOperation) throws IOException {
- // Get the volume list
- byte[] dbUserKey = metadataManager.getUserKey(owner);
- byte[] volumeList = metadataManager.get(dbUserKey);
- List<String> prevVolList = new LinkedList<>();
- if (volumeList != null) {
- VolumeList vlist = VolumeList.parseFrom(volumeList);
- prevVolList.addAll(vlist.getVolumeNamesList());
- }
-
- // Check the volume count
- if (prevVolList.size() >= maxUserVolumeCount) {
- LOG.debug("Too many volumes for user:{}", owner);
- throw new KSMException(ResultCodes.FAILED_TOO_MANY_USER_VOLUMES);
- }
-
- // Add the new volume to the list
- prevVolList.add(volume);
- VolumeList newVolList = VolumeList.newBuilder()
- .addAllVolumeNames(prevVolList).build();
- batchOperation.put(dbUserKey, newVolList.toByteArray());
- }
-
- private void delVolumeFromOwnerList(String volume, String owner,
- BatchOperation batchOperation)
- throws IOException {
- // Get the volume list
- byte[] dbUserKey = metadataManager.getUserKey(owner);
- byte[] volumeList = metadataManager.get(dbUserKey);
- List<String> prevVolList = new LinkedList<>();
- if (volumeList != null) {
- VolumeList vlist = VolumeList.parseFrom(volumeList);
- prevVolList.addAll(vlist.getVolumeNamesList());
- } else {
- LOG.debug("volume:{} not found for user:{}");
- throw new KSMException(ResultCodes.FAILED_USER_NOT_FOUND);
- }
-
- // Remove the volume from the list
- prevVolList.remove(volume);
- if (prevVolList.size() == 0) {
- batchOperation.delete(dbUserKey);
- } else {
- VolumeList newVolList = VolumeList.newBuilder()
- .addAllVolumeNames(prevVolList).build();
- batchOperation.put(dbUserKey, newVolList.toByteArray());
- }
- }
-
- /**
- * Creates a volume.
- * @param args - KsmVolumeArgs.
- */
- @Override
- public void createVolume(KsmVolumeArgs args) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(args.getVolume());
- byte[] volumeInfo = metadataManager.get(dbVolumeKey);
-
- // Check of the volume already exists
- if (volumeInfo != null) {
- LOG.debug("volume:{} already exists", args.getVolume());
- throw new KSMException(ResultCodes.FAILED_VOLUME_ALREADY_EXISTS);
- }
-
- BatchOperation batch = new BatchOperation();
- // Write the vol info
- List<OzoneProtos.KeyValue> metadataList = new LinkedList<>();
- for (Map.Entry<String, String> entry : args.getKeyValueMap().entrySet()) {
- metadataList.add(OzoneProtos.KeyValue.newBuilder()
- .setKey(entry.getKey()).setValue(entry.getValue()).build());
- }
- List<OzoneAclInfo> aclList = args.getAclMap().ozoneAclGetProtobuf();
-
- VolumeInfo newVolumeInfo = VolumeInfo.newBuilder()
- .setAdminName(args.getAdminName())
- .setOwnerName(args.getOwnerName())
- .setVolume(args.getVolume())
- .setQuotaInBytes(args.getQuotaInBytes())
- .addAllMetadata(metadataList)
- .addAllVolumeAcls(aclList)
- .setCreationTime(Time.now())
- .build();
- batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
-
- // Add volume to user list
- addVolumeToOwnerList(args.getVolume(), args.getOwnerName(), batch);
- metadataManager.writeBatch(batch);
- LOG.debug("created volume:{} user:{}", args.getVolume(),
- args.getOwnerName());
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Volume creation failed for user:{} volume:{}",
- args.getOwnerName(), args.getVolume(), ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Changes the owner of a volume.
- *
- * @param volume - Name of the volume.
- * @param owner - Name of the owner.
- * @throws IOException
- */
- @Override
- public void setOwner(String volume, String owner) throws IOException {
- Preconditions.checkNotNull(volume);
- Preconditions.checkNotNull(owner);
- metadataManager.writeLock().lock();
- try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.get(dbVolumeKey);
- if (volInfo == null) {
- LOG.debug("Changing volume ownership failed for user:{} volume:{}",
- owner, volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-
- BatchOperation batch = new BatchOperation();
- delVolumeFromOwnerList(volume, volumeArgs.getOwnerName(), batch);
- addVolumeToOwnerList(volume, owner, batch);
-
- KsmVolumeArgs newVolumeArgs =
- KsmVolumeArgs.newBuilder().setVolume(volumeArgs.getVolume())
- .setAdminName(volumeArgs.getAdminName())
- .setOwnerName(owner)
- .setQuotaInBytes(volumeArgs.getQuotaInBytes())
- .setCreationTime(volumeArgs.getCreationTime())
- .build();
-
- VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
- batch.put(dbVolumeKey, newVolumeInfo.toByteArray());
-
- metadataManager.writeBatch(batch);
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Changing volume ownership failed for user:{} volume:{}",
- owner, volume, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Changes the Quota on a volume.
- *
- * @param volume - Name of the volume.
- * @param quota - Quota in bytes.
- * @throws IOException
- */
- public void setQuota(String volume, long quota) throws IOException {
- Preconditions.checkNotNull(volume);
- metadataManager.writeLock().lock();
- try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.get(dbVolumeKey);
- if (volInfo == null) {
- LOG.debug("volume:{} does not exist", volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
-
- KsmVolumeArgs newVolumeArgs =
- KsmVolumeArgs.newBuilder()
- .setVolume(volumeArgs.getVolume())
- .setAdminName(volumeArgs.getAdminName())
- .setOwnerName(volumeArgs.getOwnerName())
- .setQuotaInBytes(quota)
- .setCreationTime(volumeArgs.getCreationTime()).build();
-
- VolumeInfo newVolumeInfo = newVolumeArgs.getProtobuf();
- metadataManager.put(dbVolumeKey, newVolumeInfo.toByteArray());
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Changing volume quota failed for volume:{} quota:{}", volume,
- quota, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Gets the volume information.
- * @param volume - Volume name.
- * @return VolumeArgs or exception is thrown.
- * @throws IOException
- */
- public KsmVolumeArgs getVolumeInfo(String volume) throws IOException {
- Preconditions.checkNotNull(volume);
- metadataManager.readLock().lock();
- try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.get(dbVolumeKey);
- if (volInfo == null) {
- LOG.debug("volume:{} does not exist", volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
- return volumeArgs;
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.warn("Info volume failed for volume:{}", volume, ex);
- }
- throw ex;
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- /**
- * Deletes an existing empty volume.
- *
- * @param volume - Name of the volume.
- * @throws IOException
- */
- @Override
- public void deleteVolume(String volume) throws IOException {
- Preconditions.checkNotNull(volume);
- metadataManager.writeLock().lock();
- try {
- BatchOperation batch = new BatchOperation();
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.get(dbVolumeKey);
- if (volInfo == null) {
- LOG.debug("volume:{} does not exist", volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- if (!metadataManager.isVolumeEmpty(volume)) {
- LOG.debug("volume:{} is not empty", volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_EMPTY);
- }
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
- // delete the volume from the owner list
- // as well as delete the volume entry
- delVolumeFromOwnerList(volume, volumeInfo.getOwnerName(), batch);
- batch.delete(dbVolumeKey);
- metadataManager.writeBatch(batch);
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Delete volume failed for volume:{}", volume, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Checks if the specified user with a role can access this volume.
- *
- * @param volume - volume
- * @param userAcl - user acl which needs to be checked for access
- * @return true if the user has access for the volume, false otherwise
- * @throws IOException
- */
- public boolean checkVolumeAccess(String volume, OzoneAclInfo userAcl)
- throws IOException {
- Preconditions.checkNotNull(volume);
- Preconditions.checkNotNull(userAcl);
- metadataManager.readLock().lock();
- try {
- byte[] dbVolumeKey = metadataManager.getVolumeKey(volume);
- byte[] volInfo = metadataManager.get(dbVolumeKey);
- if (volInfo == null) {
- LOG.debug("volume:{} does not exist", volume);
- throw new KSMException(ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- VolumeInfo volumeInfo = VolumeInfo.parseFrom(volInfo);
- KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(volumeInfo);
- Preconditions.checkState(volume.equals(volumeInfo.getVolume()));
- return volumeArgs.getAclMap().hasAccess(userAcl);
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Check volume access failed for volume:{} user:{} rights:{}",
- volume, userAcl.getName(), userAcl.getRights(), ex);
- }
- throw ex;
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KsmVolumeArgs> listVolumes(String userName,
- String prefix, String startKey, int maxKeys) throws IOException {
- metadataManager.readLock().lock();
- try {
- return metadataManager.listVolumes(
- userName, prefix, startKey, maxKeys);
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
deleted file mode 100644
index e2f3580..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/KSMException.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.exceptions;
-
-import java.io.IOException;
-
-/**
- * Exception thrown by KSM.
- */
-public class KSMException extends IOException {
- private final KSMException.ResultCodes result;
-
- /**
- * Constructs an {@code IOException} with {@code null}
- * as its error detail message.
- */
- public KSMException(KSMException.ResultCodes result) {
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- */
- public KSMException(String message, KSMException.ResultCodes result) {
- super(message);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message
- * and cause.
- * <p>
- * <p> Note that the detail message associated with {@code cause} is
- * <i>not</i> automatically incorporated into this exception's detail
- * message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the
- * {@link #getMessage()} method)
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @since 1.6
- */
- public KSMException(String message, Throwable cause,
- KSMException.ResultCodes result) {
- super(message, cause);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified cause and a
- * detail message of {@code (cause==null ? null : cause.toString())}
- * (which typically contains the class and detail message of {@code cause}).
- * This constructor is useful for IO exceptions that are little more
- * than wrappers for other throwables.
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @since 1.6
- */
- public KSMException(Throwable cause, KSMException.ResultCodes result) {
- super(cause);
- this.result = result;
- }
-
- /**
- * Returns resultCode.
- * @return ResultCode
- */
- public KSMException.ResultCodes getResult() {
- return result;
- }
-
- /**
- * Error codes to make it easy to decode these exceptions.
- */
- public enum ResultCodes {
- FAILED_TOO_MANY_USER_VOLUMES,
- FAILED_VOLUME_ALREADY_EXISTS,
- FAILED_VOLUME_NOT_FOUND,
- FAILED_VOLUME_NOT_EMPTY,
- FAILED_USER_NOT_FOUND,
- FAILED_BUCKET_ALREADY_EXISTS,
- FAILED_BUCKET_NOT_FOUND,
- FAILED_BUCKET_NOT_EMPTY,
- FAILED_KEY_ALREADY_EXISTS,
- FAILED_KEY_NOT_FOUND,
- FAILED_KEY_ALLOCATION,
- FAILED_KEY_DELETION,
- FAILED_METADATA_ERROR,
- FAILED_INTERNAL_ERROR,
- KSM_NOT_INITIALIZED,
- SCM_VERSION_MISMATCH_ERROR
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
deleted file mode 100644
index 09fd87f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/exceptions/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm.exceptions;
-// Exception thrown by KSM.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
deleted file mode 100644
index 3eb6dc1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManager.java
+++ /dev/null
@@ -1,1260 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import com.google.protobuf.BlockingService;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.jmx.ServiceRuntimeInfoImpl;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.Storage.StorageState;
-import org.apache.hadoop.ozone.common.StorageInfo;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.commands.DeleteBlocksCommand;
-import org.apache.hadoop.ozone.protocol.commands.RegisteredCommand;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMCommandResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeAddressList;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMReregisterCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SendContainerReportProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.Type;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto.DeleteBlockTransactionResult;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocolPB.ScmBlockLocationProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerLocationProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ozone.scm.block.BlockManager;
-import org.apache.hadoop.ozone.scm.block.BlockManagerImpl;
-import org.apache.hadoop.ozone.scm.container.ContainerMapping;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.ContainerStat;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMMetrics;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.SCMNodeManager;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.TreeSet;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_DATANODE_ADDRESS_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DB_CACHE_SIZE_MB;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.util.ExitUtil.terminate;
-
-/**
- * StorageContainerManager is the main entry point for the service that provides
- * information about which SCM nodes host containers.
- *
- * DataNodes report to StorageContainerManager using heartbeat
- * messages. SCM allocates containers and returns a pipeline.
- *
- * A client once it gets a pipeline (a list of datanodes) will connect to the
- * datanodes and create a container, which then can be used to store data.
- */
-@InterfaceAudience.LimitedPrivate({"HDFS", "CBLOCK", "OZONE", "HBASE"})
-public class StorageContainerManager extends ServiceRuntimeInfoImpl
- implements StorageContainerDatanodeProtocol,
- StorageContainerLocationProtocol, ScmBlockLocationProtocol, SCMMXBean {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(StorageContainerManager.class);
-
- /**
- * Startup options.
- */
- public enum StartupOption {
- INIT("-init"),
- CLUSTERID("-clusterid"),
- GENCLUSTERID("-genclusterid"),
- REGULAR("-regular"),
- HELP("-help");
-
- private final String name;
- private String clusterId = null;
-
- public void setClusterId(String cid) {
- if(cid != null && !cid.isEmpty()) {
- clusterId = cid;
- }
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- StartupOption(String arg) {
- this.name = arg;
- }
-
- public String getName() {
- return name;
- }
- }
-
- /**
- * NodeManager and container Managers for SCM.
- */
- private final NodeManager scmNodeManager;
- private final Mapping scmContainerManager;
- private final BlockManager scmBlockManager;
- private final SCMStorage scmStorage;
-
- /** The RPC server that listens to requests from DataNodes. */
- private final RPC.Server datanodeRpcServer;
- private final InetSocketAddress datanodeRpcAddress;
-
- /** The RPC server that listens to requests from clients. */
- private final RPC.Server clientRpcServer;
- private final InetSocketAddress clientRpcAddress;
-
- /** The RPC server that listens to requests from block service clients. */
- private final RPC.Server blockRpcServer;
- private final InetSocketAddress blockRpcAddress;
-
- private final StorageContainerManagerHttpServer httpServer;
-
- /** SCM mxbean. */
- private ObjectName scmInfoBeanName;
-
- /** SCM super user. */
- private final String scmUsername;
- private final Collection<String> scmAdminUsernames;
-
- /** SCM metrics. */
- private static SCMMetrics metrics;
- /** Key = DatanodeUuid, value = ContainerStat. */
- private Cache<String, ContainerStat> containerReportCache;
-
-
- private static final String USAGE =
- "Usage: \n hdfs scm [genericOptions] "
- + "[ " + StartupOption.INIT.getName() + " [ "
- + StartupOption.CLUSTERID.getName() + " <cid> ] ]\n "
- + "hdfs scm [genericOptions] [ "
- + StartupOption.GENCLUSTERID.getName() + " ]\n " +
- "hdfs scm [ "
- + StartupOption.HELP.getName() + " ]\n";
- /**
- * Creates a new StorageContainerManager. Configuration will be updated with
- * information on the actual listening addresses used for RPC servers.
- *
- * @param conf configuration
- */
- private StorageContainerManager(OzoneConfiguration conf)
- throws IOException {
-
- final int handlerCount = conf.getInt(
- OZONE_SCM_HANDLER_COUNT_KEY, OZONE_SCM_HANDLER_COUNT_DEFAULT);
- final int cacheSize = conf.getInt(OZONE_SCM_DB_CACHE_SIZE_MB,
- OZONE_SCM_DB_CACHE_SIZE_DEFAULT);
-
- StorageContainerManager.initMetrics();
- initContainerReportCache(conf);
-
- scmStorage = new SCMStorage(conf);
- if (scmStorage.getState() != StorageState.INITIALIZED) {
- throw new SCMException("SCM not initialized.",
- ResultCodes.SCM_NOT_INITIALIZED);
- }
- scmNodeManager = new SCMNodeManager(conf, scmStorage.getClusterID(), this);
- scmContainerManager = new ContainerMapping(conf, scmNodeManager, cacheSize);
- scmBlockManager = new BlockManagerImpl(conf, scmNodeManager,
- scmContainerManager, cacheSize);
-
- scmAdminUsernames = conf.getTrimmedStringCollection(
- OzoneConfigKeys.OZONE_ADMINISTRATORS);
- scmUsername = UserGroupInformation.getCurrentUser().getUserName();
- if (!scmAdminUsernames.contains(scmUsername)) {
- scmAdminUsernames.add(scmUsername);
- }
-
- RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
- ProtobufRpcEngine.class);
- RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- RPC.setProtocolEngine(conf, ScmBlockLocationProtocolPB.class,
- ProtobufRpcEngine.class);
-
- BlockingService dnProtoPbService = StorageContainerDatanodeProtocolProtos.
- StorageContainerDatanodeProtocolService.newReflectiveBlockingService(
- new StorageContainerDatanodeProtocolServerSideTranslatorPB(this));
-
- final InetSocketAddress datanodeRpcAddr =
- OzoneClientUtils.getScmDataNodeBindAddress(conf);
- datanodeRpcServer = startRpcServer(conf, datanodeRpcAddr,
- StorageContainerDatanodeProtocolPB.class, dnProtoPbService,
- handlerCount);
- datanodeRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
- OZONE_SCM_DATANODE_ADDRESS_KEY, datanodeRpcAddr, datanodeRpcServer);
-
- // SCM Container Service RPC
- BlockingService storageProtoPbService =
- StorageContainerLocationProtocolProtos
- .StorageContainerLocationProtocolService
- .newReflectiveBlockingService(
- new StorageContainerLocationProtocolServerSideTranslatorPB(this));
-
- final InetSocketAddress scmAddress =
- OzoneClientUtils.getScmClientBindAddress(conf);
- clientRpcServer = startRpcServer(conf, scmAddress,
- StorageContainerLocationProtocolPB.class, storageProtoPbService,
- handlerCount);
- clientRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
- OZONE_SCM_CLIENT_ADDRESS_KEY, scmAddress, clientRpcServer);
-
- // SCM Block Service RPC
- BlockingService blockProtoPbService =
- ScmBlockLocationProtocolProtos
- .ScmBlockLocationProtocolService
- .newReflectiveBlockingService(
- new ScmBlockLocationProtocolServerSideTranslatorPB(this));
-
- final InetSocketAddress scmBlockAddress =
- OzoneClientUtils.getScmBlockClientBindAddress(conf);
- blockRpcServer = startRpcServer(conf, scmBlockAddress,
- ScmBlockLocationProtocolPB.class, blockProtoPbService,
- handlerCount);
- blockRpcAddress = OzoneClientUtils.updateRPCListenAddress(conf,
- OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY, scmBlockAddress, blockRpcServer);
-
- httpServer = new StorageContainerManagerHttpServer(conf);
-
- registerMXBean();
- }
-
- /**
- * Initialize container reports cache that sent from datanodes.
- *
- * @param conf
- */
- private void initContainerReportCache(OzoneConfiguration conf) {
- containerReportCache = CacheBuilder.newBuilder()
- .expireAfterAccess(Long.MAX_VALUE, TimeUnit.MILLISECONDS)
- .maximumSize(Integer.MAX_VALUE)
- .removalListener(new RemovalListener<String, ContainerStat>() {
- @Override
- public void onRemoval(
- RemovalNotification<String, ContainerStat> removalNotification) {
- synchronized (containerReportCache) {
- ContainerStat stat = removalNotification.getValue();
- // remove invalid container report
- metrics.decrContainerStat(stat);
- LOG.debug(
- "Remove expired container stat entry for datanode: {}.",
- removalNotification.getKey());
- }
- }
- }).build();
- }
-
- /**
- * Builds a message for logging startup information about an RPC server.
- *
- * @param description RPC server description
- * @param addr RPC server listening address
- * @return server startup message
- */
- private static String buildRpcServerStartMessage(String description,
- InetSocketAddress addr) {
- return addr != null ? String.format("%s is listening at %s",
- description, addr.toString()) :
- String.format("%s not started", description);
- }
-
- /**
- * Starts an RPC server, if configured.
- *
- * @param conf configuration
- * @param addr configured address of RPC server
- * @param protocol RPC protocol provided by RPC server
- * @param instance RPC protocol implementation instance
- * @param handlerCount RPC server handler count
- *
- * @return RPC server
- * @throws IOException if there is an I/O error while creating RPC server
- */
- private static RPC.Server startRpcServer(OzoneConfiguration conf,
- InetSocketAddress addr, Class<?> protocol, BlockingService instance,
- int handlerCount)
- throws IOException {
- RPC.Server rpcServer = new RPC.Builder(conf)
- .setProtocol(protocol)
- .setInstance(instance)
- .setBindAddress(addr.getHostString())
- .setPort(addr.getPort())
- .setNumHandlers(handlerCount)
- .setVerbose(false)
- .setSecretManager(null)
- .build();
-
- DFSUtil.addPBProtocol(conf, protocol, instance, rpcServer);
- return rpcServer;
- }
-
- private void registerMXBean() {
- Map<String, String> jmxProperties = new HashMap<>();
- jmxProperties.put("component", "ServerRuntime");
- this.scmInfoBeanName =
- MBeans.register("StorageContainerManager",
- "StorageContainerManagerInfo",
- jmxProperties,
- this);
- }
-
- private void unregisterMXBean() {
- if(this.scmInfoBeanName != null) {
- MBeans.unregister(this.scmInfoBeanName);
- this.scmInfoBeanName = null;
- }
- }
-
- /**
- * Main entry point for starting StorageContainerManager.
- *
- * @param argv arguments
- * @throws IOException if startup fails due to I/O error
- */
- public static void main(String[] argv) throws IOException {
- if (DFSUtil.parseHelpArgument(argv, USAGE,
- System.out, true)) {
- System.exit(0);
- }
- try {
- OzoneConfiguration conf = new OzoneConfiguration();
- GenericOptionsParser hParser = new GenericOptionsParser(conf, argv);
- if (!hParser.isParseSuccessful()) {
- System.err.println("USAGE: " + USAGE + "\n");
- hParser.printGenericCommandUsage(System.err);
- System.exit(1);
- }
- StringUtils.startupShutdownMessage(StorageContainerManager.class,
- argv, LOG);
- StorageContainerManager scm = createSCM(hParser.getRemainingArgs(), conf);
- if (scm != null) {
- scm.start();
- scm.join();
- }
- } catch (Throwable t) {
- LOG.error("Failed to start the StorageContainerManager.", t);
- terminate(1, t);
- }
- }
-
- private static void printUsage(PrintStream out) {
- out.println(USAGE + "\n");
- }
-
- public static StorageContainerManager createSCM(String[] argv,
- OzoneConfiguration conf) throws IOException {
- if (!DFSUtil.isOzoneEnabled(conf)) {
- System.err.println("SCM cannot be started in secure mode or when " +
- OZONE_ENABLED + " is set to false");
- System.exit(1);
- }
- StartupOption startOpt = parseArguments(argv);
- if (startOpt == null) {
- printUsage(System.err);
- terminate(1);
- return null;
- }
- switch (startOpt) {
- case INIT:
- terminate(scmInit(conf) ? 0 : 1);
- return null;
- case GENCLUSTERID:
- System.out.println("Generating new cluster id:");
- System.out.println(StorageInfo.newClusterID());
- terminate(0);
- return null;
- case HELP:
- printUsage(System.err);
- terminate(0);
- return null;
- default:
- return new StorageContainerManager(conf);
- }
- }
-
- /**
- * Routine to set up the Version info for StorageContainerManager.
- *
- * @param conf OzoneConfiguration
- * @return true if SCM initialization is successful, false otherwise.
- * @throws IOException if init fails due to I/O error
- */
- public static boolean scmInit(OzoneConfiguration conf) throws IOException {
- SCMStorage scmStorage = new SCMStorage(conf);
- StorageState state = scmStorage.getState();
- if (state != StorageState.INITIALIZED) {
- try {
- String clusterId = StartupOption.INIT.getClusterId();
- if (clusterId != null && !clusterId.isEmpty()) {
- scmStorage.setClusterId(clusterId);
- }
- scmStorage.initialize();
- System.out.println("SCM initialization succeeded." +
- "Current cluster id for sd=" + scmStorage.getStorageDir() + ";cid="
- + scmStorage.getClusterID());
- return true;
- } catch (IOException ioe) {
- LOG.error("Could not initialize SCM version file", ioe);
- return false;
- }
- } else {
- System.out.println("SCM already initialized. Reusing existing" +
- " cluster id for sd=" + scmStorage.getStorageDir() + ";cid="
- + scmStorage.getClusterID());
- return true;
- }
- }
-
- private static StartupOption parseArguments(String[] args) {
- int argsLen = (args == null) ? 0 : args.length;
- StartupOption startOpt = StartupOption.HELP;
- if (argsLen == 0) {
- startOpt = StartupOption.REGULAR;
- }
- for (int i = 0; i < argsLen; i++) {
- String cmd = args[i];
- if (StartupOption.INIT.getName().equalsIgnoreCase(cmd)) {
- startOpt = StartupOption.INIT;
- if (argsLen > 3) {
- return null;
- }
- for (i = i + 1; i < argsLen; i++) {
- if (args[i].equalsIgnoreCase(StartupOption.CLUSTERID.getName())) {
- i++;
- if (i < argsLen && !args[i].isEmpty()) {
- startOpt.setClusterId(args[i]);
- } else {
- // if no cluster id specified or is empty string, return null
- LOG.error("Must specify a valid cluster ID after the "
- + StartupOption.CLUSTERID.getName() + " flag");
- return null;
- }
- } else {
- return null;
- }
- }
- } else if (StartupOption.GENCLUSTERID.getName().equalsIgnoreCase(cmd)) {
- if (argsLen > 1) {
- return null;
- }
- startOpt = StartupOption.GENCLUSTERID;
- }
- }
- return startOpt;
- }
-
- /**
- * Returns a SCMCommandRepose from the SCM Command.
- * @param cmd - Cmd
- * @return SCMCommandResponseProto
- * @throws InvalidProtocolBufferException
- */
- @VisibleForTesting
- public SCMCommandResponseProto getCommandResponse(SCMCommand cmd,
- final String datanodID)
- throws IOException {
- Type type = cmd.getType();
- SCMCommandResponseProto.Builder builder =
- SCMCommandResponseProto.newBuilder()
- .setDatanodeUUID(datanodID);
- switch (type) {
- case registeredCommand:
- return builder.setCmdType(Type.registeredCommand)
- .setRegisteredProto(
- SCMRegisteredCmdResponseProto.getDefaultInstance())
- .build();
- case versionCommand:
- return builder.setCmdType(Type.versionCommand)
- .setVersionProto(SCMVersionResponseProto.getDefaultInstance())
- .build();
- case sendContainerReport:
- return builder.setCmdType(Type.sendContainerReport)
- .setSendReport(SendContainerReportProto.getDefaultInstance())
- .build();
- case reregisterCommand:
- return builder.setCmdType(Type.reregisterCommand)
- .setReregisterProto(SCMReregisterCmdResponseProto
- .getDefaultInstance())
- .build();
- case deleteBlocksCommand:
- // Once SCM sends out the deletion message, increment the count.
- // this is done here instead of when SCM receives the ACK, because
- // DN might not be able to response the ACK for sometime. In case
- // it times out, SCM needs to re-send the message some more times.
- List<Long> txs = ((DeleteBlocksCommand) cmd).blocksTobeDeleted()
- .stream().map(tx -> tx.getTxID()).collect(Collectors.toList());
- this.getScmBlockManager().getDeletedBlockLog().incrementCount(txs);
- return builder.setCmdType(Type.deleteBlocksCommand)
- .setDeleteBlocksProto(((DeleteBlocksCommand) cmd).getProto())
- .build();
- case closeContainerCommand:
- return builder.setCmdType(Type.closeContainerCommand)
- .setCloseContainerProto(((CloseContainerCommand)cmd).getProto())
- .build();
- default:
- throw new IllegalArgumentException("Not implemented");
- }
- }
-
- @VisibleForTesting
- public static SCMRegisteredCmdResponseProto getRegisteredResponse(
- SCMCommand cmd, SCMNodeAddressList addressList) {
- Preconditions.checkState(cmd.getClass() == RegisteredCommand.class);
- RegisteredCommand rCmd = (RegisteredCommand) cmd;
- StorageContainerDatanodeProtocolProtos.Type type = cmd.getType();
- if (type != Type.registeredCommand) {
- throw new IllegalArgumentException("Registered command is not well " +
- "formed. Internal Error.");
- }
- return SCMRegisteredCmdResponseProto.newBuilder()
- //TODO : Fix this later when we have multiple SCM support.
- //.setAddressList(addressList)
- .setErrorCode(rCmd.getError())
- .setClusterID(rCmd.getClusterID())
- .setDatanodeUUID(rCmd.getDatanodeUUID()).build();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public Pipeline getContainer(String containerName) throws IOException {
- checkAdminAccess();
- return scmContainerManager.getContainer(containerName).getPipeline();
- }
-
- @VisibleForTesting
- public ContainerInfo getContainerInfo(String containerName)
- throws IOException {
- return scmContainerManager.getContainer(containerName);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName,
- String prefixName, int count) throws IOException {
- return scmContainerManager.listContainer(startName, prefixName, count);
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void deleteContainer(String containerName) throws IOException {
- checkAdminAccess();
- scmContainerManager.deleteContainer(containerName);
- }
-
- /**
- * Queries a list of Node Statuses.
- *
- * @param nodeStatuses
- * @param queryScope
- * @param poolName @return List of Datanodes.
- */
- @Override
- public OzoneProtos.NodePool queryNode(EnumSet<NodeState> nodeStatuses,
- OzoneProtos.QueryScope queryScope, String poolName) throws IOException {
-
- if (queryScope == OzoneProtos.QueryScope.POOL) {
- throw new IllegalArgumentException("Not Supported yet");
- }
-
- List<DatanodeID> datanodes = queryNode(nodeStatuses);
- OzoneProtos.NodePool.Builder poolBuilder =
- OzoneProtos.NodePool.newBuilder();
-
- for (DatanodeID datanode : datanodes) {
- OzoneProtos.Node node = OzoneProtos.Node.newBuilder()
- .setNodeID(datanode.getProtoBufMessage())
- .addAllNodeStates(nodeStatuses)
- .build();
- poolBuilder.addNodes(node);
- }
-
- return poolBuilder.build();
- }
-
- /**
- * Notify from client when begin/finish operation for container/pipeline
- * objects on datanodes.
- * @param type
- * @param name
- * @param op
- * @param stage
- */
- @Override
- public void notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type type, String name,
- ObjectStageChangeRequestProto.Op op,
- ObjectStageChangeRequestProto.Stage stage) throws IOException {
-
- LOG.info("Object type {} name {} op {} new stage {}",
- type, name, op, stage);
- if (type == ObjectStageChangeRequestProto.Type.container) {
- if (op == ObjectStageChangeRequestProto.Op.create) {
- if (stage == ObjectStageChangeRequestProto.Stage.begin) {
- scmContainerManager.updateContainerState(name,
- OzoneProtos.LifeCycleEvent.CREATE);
- } else {
- scmContainerManager.updateContainerState(name,
- OzoneProtos.LifeCycleEvent.CREATED);
- }
- } else if (op == ObjectStageChangeRequestProto.Op.close) {
- if (stage == ObjectStageChangeRequestProto.Stage.begin) {
- scmContainerManager.updateContainerState(name,
- OzoneProtos.LifeCycleEvent.FINALIZE);
- } else {
- scmContainerManager.updateContainerState(name,
- OzoneProtos.LifeCycleEvent.CLOSE);
- }
- }
- } //else if (type == ObjectStageChangeRequestProto.Type.pipeline) {
- // TODO: pipeline state update will be addressed in future patch.
- //}
- }
-
- /**
- * Creates a replication pipeline of a specified type.
- */
- @Override
- public Pipeline createReplicationPipeline(
- OzoneProtos.ReplicationType replicationType,
- OzoneProtos.ReplicationFactor factor,
- OzoneProtos.NodePool nodePool)
- throws IOException {
- // TODO: will be addressed in future patch.
- return null;
- }
-
- /**
- * Queries a list of Node that match a set of statuses.
- * <p>
- * For example, if the nodeStatuses is HEALTHY and RAFT_MEMBER,
- * then this call will return all healthy nodes which members in
- * Raft pipeline.
- * <p>
- * Right now we don't support operations, so we assume it is an AND operation
- * between the operators.
- *
- * @param nodeStatuses - A set of NodeStates.
- * @return List of Datanodes.
- */
-
- public List<DatanodeID> queryNode(EnumSet<NodeState> nodeStatuses) {
- Preconditions.checkNotNull(nodeStatuses, "Node Query set cannot be null");
- Preconditions.checkState(nodeStatuses.size() > 0, "No valid arguments " +
- "in the query set");
- List<DatanodeID> resultList = new LinkedList<>();
- Set<DatanodeID> currentSet = new TreeSet<>();
-
- for (NodeState nodeState : nodeStatuses) {
- Set<DatanodeID> nextSet = queryNodeState(nodeState);
- if ((nextSet == null) || (nextSet.size() == 0)) {
- // Right now we only support AND operation. So intersect with
- // any empty set is null.
- return resultList;
- }
- // First time we have to add all the elements, next time we have to
- // do an intersection operation on the set.
- if (currentSet.size() == 0) {
- currentSet.addAll(nextSet);
- } else {
- currentSet.retainAll(nextSet);
- }
- }
-
- resultList.addAll(currentSet);
- return resultList;
- }
-
- /**
- * Query the System for Nodes.
- *
- * @param nodeState - NodeState that we are interested in matching.
- * @return Set of Datanodes that match the NodeState.
- */
- private Set<DatanodeID> queryNodeState(NodeState nodeState) {
- if (nodeState == NodeState.RAFT_MEMBER ||
- nodeState == NodeState.FREE_NODE) {
- throw new IllegalStateException("Not implemented yet");
- }
- Set<DatanodeID> returnSet = new TreeSet<>();
- List<DatanodeID> tmp = getScmNodeManager().getNodes(nodeState);
- if ((tmp != null) && (tmp.size() > 0)) {
- returnSet.addAll(tmp);
- }
- return returnSet;
- }
-
- /**
- * Asks SCM where a container should be allocated. SCM responds with the set
- * of datanodes that should be used creating this container.
- *
- * @param containerName - Name of the container.
- * @param replicationFactor - replication factor.
- * @return pipeline
- * @throws IOException
- */
- @Override
- public Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType,
- OzoneProtos.ReplicationFactor replicationFactor, String containerName,
- String owner) throws IOException {
-
- checkAdminAccess();
- return scmContainerManager
- .allocateContainer(replicationType, replicationFactor, containerName,
- owner).getPipeline();
- }
-
- /**
- * Returns listening address of StorageLocation Protocol RPC server.
- *
- * @return listen address of StorageLocation RPC server
- */
- @VisibleForTesting
- public InetSocketAddress getClientRpcAddress() {
- return clientRpcAddress;
- }
-
- @Override
- public String getClientRpcPort() {
- InetSocketAddress addr = getClientRpcAddress();
- return addr == null ? "0" : Integer.toString(addr.getPort());
- }
-
- /**
- * Returns listening address of StorageDatanode Protocol RPC server.
- *
- * @return Address where datanode are communicating.
- */
- public InetSocketAddress getDatanodeRpcAddress() {
- return datanodeRpcAddress;
- }
-
- @Override
- public String getDatanodeRpcPort() {
- InetSocketAddress addr = getDatanodeRpcAddress();
- return addr == null ? "0" : Integer.toString(addr.getPort());
- }
-
- /**
- * Start service.
- */
- public void start() throws IOException {
- LOG.info(buildRpcServerStartMessage(
- "StorageContainerLocationProtocol RPC server", clientRpcAddress));
- DefaultMetricsSystem.initialize("StorageContainerManager");
- clientRpcServer.start();
- LOG.info(buildRpcServerStartMessage(
- "ScmBlockLocationProtocol RPC server", blockRpcAddress));
- blockRpcServer.start();
- LOG.info(buildRpcServerStartMessage("RPC server for DataNodes",
- datanodeRpcAddress));
- datanodeRpcServer.start();
- httpServer.start();
- scmBlockManager.start();
-
- setStartTime();
-
- }
-
- /**
- * Stop service.
- */
- public void stop() {
- try {
- LOG.info("Stopping block service RPC server");
- blockRpcServer.stop();
- } catch (Exception ex) {
- LOG.error("Storage Container Manager blockRpcServer stop failed.", ex);
- }
-
- try {
- LOG.info("Stopping the StorageContainerLocationProtocol RPC server");
- clientRpcServer.stop();
- } catch (Exception ex) {
- LOG.error("Storage Container Manager clientRpcServer stop failed.", ex);
- }
-
- try {
- LOG.info("Stopping the RPC server for DataNodes");
- datanodeRpcServer.stop();
- } catch (Exception ex) {
- LOG.error("Storage Container Manager datanodeRpcServer stop failed.", ex);
- }
-
- try {
- LOG.info("Stopping Storage Container Manager HTTP server.");
- httpServer.stop();
- } catch (Exception ex) {
- LOG.error("Storage Container Manager HTTP server stop failed.", ex);
- }
-
- try {
- LOG.info("Stopping Block Manager Service.");
- scmBlockManager.stop();
- } catch (Exception ex) {
- LOG.error("SCM block manager service stop failed.", ex);
- }
-
- if (containerReportCache != null) {
- containerReportCache.invalidateAll();
- containerReportCache.cleanUp();
- }
-
- if (metrics != null) {
- metrics.unRegister();
- }
-
- unregisterMXBean();
- IOUtils.cleanupWithLogger(LOG, scmContainerManager);
- IOUtils.cleanupWithLogger(LOG, scmNodeManager);
- }
-
- /**
- * Wait until service has completed shutdown.
- */
- public void join() {
- try {
- blockRpcServer.join();
- clientRpcServer.join();
- datanodeRpcServer.join();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.info("Interrupted during StorageContainerManager join.");
- }
- }
-
- /**
- * Returns SCM version.
- *
- * @return Version info.
- */
- @Override
- public SCMVersionResponseProto getVersion(
- SCMVersionRequestProto versionRequest) throws IOException {
- return getScmNodeManager().getVersion(versionRequest).getProtobufMessage();
- }
-
- /**
- * Used by data node to send a Heartbeat.
- *
- * @param datanodeID - Datanode ID.
- * @param nodeReport - Node Report
- * @param reportState - Container report ready info.
- * @return - SCMHeartbeatResponseProto
- * @throws IOException
- */
- @Override
- public SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
- SCMNodeReport nodeReport, ReportState reportState) throws IOException {
- List<SCMCommand> commands =
- getScmNodeManager().sendHeartbeat(datanodeID, nodeReport, reportState);
- List<SCMCommandResponseProto> cmdResponses = new LinkedList<>();
- for (SCMCommand cmd : commands) {
- cmdResponses.add(getCommandResponse(cmd, datanodeID.getDatanodeUuid()
- .toString()));
- }
- return SCMHeartbeatResponseProto.newBuilder().addAllCommands(cmdResponses)
- .build();
- }
-
- /**
- * Register Datanode.
- *
- * @param datanodeID - DatanodID.
- * @param scmAddresses - List of SCMs this datanode is configured to
- * communicate.
- * @return SCM Command.
- */
- @Override
- public StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
- register(DatanodeID datanodeID, String[] scmAddresses)
- throws IOException {
- // TODO : Return the list of Nodes that forms the SCM HA.
- return getRegisteredResponse(scmNodeManager.register(datanodeID), null);
- }
-
- /**
- * Send a container report.
- *
- * @param reports -- Container report
- * @return HeartbeatRespose.nullcommand.
- * @throws IOException
- */
- @Override
- public ContainerReportsResponseProto sendContainerReport(
- ContainerReportsRequestProto reports) throws IOException {
- updateContainerReportMetrics(reports);
-
- // should we process container reports async?
- scmContainerManager.processContainerReports(reports);
- return ContainerReportsResponseProto.newBuilder().build();
- }
-
- private void updateContainerReportMetrics(
- ContainerReportsRequestProto reports) {
- ContainerStat newStat = null;
- // TODO: We should update the logic once incremental container report
- // type is supported.
- if (reports
- .getType() == ContainerReportsRequestProto.reportType.fullReport) {
- newStat = new ContainerStat();
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo info : reports
- .getReportsList()) {
- newStat.add(new ContainerStat(info.getSize(), info.getUsed(),
- info.getKeyCount(), info.getReadBytes(), info.getWriteBytes(),
- info.getReadCount(), info.getWriteCount()));
- }
-
- // update container metrics
- metrics.setLastContainerStat(newStat);
- }
-
- // Update container stat entry, this will trigger a removal operation if it
- // exists in cache.
- synchronized (containerReportCache) {
- String datanodeUuid = reports.getDatanodeID().getDatanodeUuid();
- if (datanodeUuid != null && newStat != null) {
- containerReportCache.put(datanodeUuid, newStat);
- // update global view container metrics
- metrics.incrContainerStat(newStat);
- }
- }
- }
-
- /**
- * Handles the block deletion ACKs sent by datanodes. Once ACKs recieved,
- * SCM considers the blocks are deleted and update the metadata in SCM DB.
- *
- * @param acks
- * @return
- * @throws IOException
- */
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto acks) throws IOException {
- if (acks.getResultsCount() > 0) {
- List<DeleteBlockTransactionResult> resultList = acks.getResultsList();
- for (DeleteBlockTransactionResult result : resultList) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Got block deletion ACK from datanode, TXIDs={}, "
- + "success={}", result.getTxID(), result.getSuccess());
- }
- if (result.getSuccess()) {
- LOG.debug("Purging TXID={} from block deletion log",
- result.getTxID());
- this.getScmBlockManager().getDeletedBlockLog()
- .commitTransactions(Collections.singletonList(result.getTxID()));
- } else {
- LOG.warn("Got failed ACK for TXID={}, prepare to resend the "
- + "TX in next interval", result.getTxID());
- }
- }
- }
- return ContainerBlocksDeletionACKResponseProto.newBuilder()
- .getDefaultInstanceForType();
- }
-
- /**
- * Returns the Number of Datanodes that are communicating with SCM.
- *
- * @param nodestate Healthy, Dead etc.
- * @return int -- count
- */
- public int getNodeCount(NodeState nodestate) {
- return scmNodeManager.getNodeCount(nodestate);
- }
-
- /**
- * Returns SCM container manager.
- */
- @VisibleForTesting
- public Mapping getScmContainerManager() {
- return scmContainerManager;
- }
-
- /**
- * Returns node manager.
- * @return - Node Manager
- */
- @VisibleForTesting
- public NodeManager getScmNodeManager() {
- return scmNodeManager;
- }
-
- @VisibleForTesting
- public BlockManager getScmBlockManager() {
- return scmBlockManager;
- }
-
- /**
- * Get block locations.
- * @param keys batch of block keys to retrieve.
- * @return set of allocated blocks.
- * @throws IOException
- */
- @Override
- public Set<AllocatedBlock> getBlockLocations(final Set<String> keys)
- throws IOException {
- Set<AllocatedBlock> locatedBlocks = new HashSet<>();
- for (String key: keys) {
- Pipeline pipeline = scmBlockManager.getBlock(key);
- AllocatedBlock block = new AllocatedBlock.Builder()
- .setKey(key)
- .setPipeline(pipeline).build();
- locatedBlocks.add(block);
- }
- return locatedBlocks;
- }
-
- /**
- * Asks SCM where a block should be allocated. SCM responds with the set of
- * datanodes that should be used creating this block.
- *
- * @param size - size of the block.
- * @param type - Replication type.
- * @param factor
- * @return allocated block accessing info (key, pipeline).
- * @throws IOException
- */
- @Override
- public AllocatedBlock allocateBlock(long size,
- OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor,
- String owner) throws IOException {
- return scmBlockManager.allocateBlock(size, type, factor, owner);
- }
-
- /**
- * Get the clusterId and SCM Id from the version file in SCM.
- */
- @Override
- public ScmInfo getScmInfo() throws IOException {
- ScmInfo.Builder builder = new ScmInfo.Builder()
- .setClusterId(scmStorage.getClusterID())
- .setScmId(scmStorage.getScmId());
- return builder.build();
- }
- /**
- * Delete blocks for a set of object keys.
- *
- * @param keyBlocksInfoList list of block keys with object keys to delete.
- * @return deletion results.
- */
- public List<DeleteBlockGroupResult> deleteKeyBlocks(
- List<BlockGroup> keyBlocksInfoList) throws IOException {
- LOG.info("SCM is informed by KSM to delete {} blocks",
- keyBlocksInfoList.size());
- List<DeleteBlockGroupResult> results = new ArrayList<>();
- for (BlockGroup keyBlocks : keyBlocksInfoList) {
- Result resultCode;
- try {
- // We delete blocks in an atomic operation to prevent getting
- // into state like only a partial of blocks are deleted,
- // which will leave key in an inconsistent state.
- scmBlockManager.deleteBlocks(keyBlocks.getBlockIDList());
- resultCode = Result.success;
- } catch (SCMException scmEx) {
- LOG.warn("Fail to delete block: {}", keyBlocks.getGroupID(), scmEx);
- switch (scmEx.getResult()) {
- case CHILL_MODE_EXCEPTION:
- resultCode = Result.chillMode;
- break;
- case FAILED_TO_FIND_BLOCK:
- resultCode = Result.errorNotFound;
- break;
- default:
- resultCode = Result.unknownFailure;
- }
- } catch (IOException ex) {
- LOG.warn("Fail to delete blocks for object key: {}",
- keyBlocks.getGroupID(), ex);
- resultCode = Result.unknownFailure;
- }
- List<DeleteBlockResult> blockResultList = new ArrayList<>();
- for (String blockKey : keyBlocks.getBlockIDList()) {
- blockResultList.add(new DeleteBlockResult(blockKey, resultCode));
- }
- results.add(new DeleteBlockGroupResult(keyBlocks.getGroupID(),
- blockResultList));
- }
- return results;
- }
-
- @VisibleForTesting
- public String getPpcRemoteUsername() {
- UserGroupInformation user = ProtobufRpcEngine.Server.getRemoteUser();
- return user == null ? null : user.getUserName();
- }
-
- private void checkAdminAccess() throws IOException {
- String remoteUser = getPpcRemoteUsername();
- if(remoteUser != null) {
- if (!scmAdminUsernames.contains(remoteUser)) {
- throw new IOException(
- "Access denied for user " + remoteUser
- + ". Superuser privilege is required.");
- }
- }
- }
-
- /**
- * Initialize SCM metrics.
- */
- public static void initMetrics() {
- metrics = SCMMetrics.create();
- }
-
- /**
- * Return SCM metrics instance.
- */
- public static SCMMetrics getMetrics() {
- return metrics == null ? SCMMetrics.create() : metrics;
- }
-
- /**
- * Invalidate container stat entry for given datanode.
- *
- * @param datanodeUuid
- */
- public void removeContainerReport(String datanodeUuid) {
- synchronized (containerReportCache) {
- containerReportCache.invalidate(datanodeUuid);
- }
- }
-
- /**
- * Get container stat of specified datanode.
- *
- * @param datanodeUuid
- * @return
- */
- public ContainerStat getContainerReport(String datanodeUuid) {
- ContainerStat stat = null;
- synchronized (containerReportCache) {
- stat = containerReportCache.getIfPresent(datanodeUuid);
- }
-
- return stat;
- }
-
- /**
- * Returns a view of the container stat entries. Modifications made to the
- * map will directly affect the cache.
- *
- * @return
- */
- public ConcurrentMap<String, ContainerStat> getContainerReportCache() {
- return containerReportCache.asMap();
- }
-
- @Override
- public Map<String, String> getContainerReport() {
- Map<String, String> id2StatMap = new HashMap<>();
- synchronized (containerReportCache) {
- ConcurrentMap<String, ContainerStat> map = containerReportCache.asMap();
- for (Map.Entry<String, ContainerStat> entry : map.entrySet()) {
- id2StatMap.put(entry.getKey(), entry.getValue().toJsonString());
- }
- }
-
- return id2StatMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
deleted file mode 100644
index 62c871c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/StorageContainerManagerHttpServer.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.web.OzoneHttpServer;
-import org.apache.hadoop.scm.ScmConfigKeys;
-
-import java.io.IOException;
-
-/**
- * HttpServer2 wrapper for the Ozone Storage Container Manager.
- */
-public class StorageContainerManagerHttpServer extends OzoneHttpServer {
-
- public StorageContainerManagerHttpServer(Configuration conf)
- throws IOException {
- super(conf, "scm");
- }
-
- @Override protected String getHttpAddressKey() {
- return ScmConfigKeys.OZONE_SCM_HTTP_ADDRESS_KEY;
- }
-
- @Override protected String getHttpBindHostKey() {
- return ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_KEY;
- }
-
- @Override protected String getHttpsAddressKey() {
- return ScmConfigKeys.OZONE_SCM_HTTPS_ADDRESS_KEY;
- }
-
- @Override protected String getHttpsBindHostKey() {
- return ScmConfigKeys.OZONE_SCM_HTTPS_BIND_HOST_KEY;
- }
-
- @Override protected String getBindHostDefault() {
- return ScmConfigKeys.OZONE_SCM_HTTP_BIND_HOST_DEFAULT;
- }
-
- @Override protected int getHttpBindPortDefault() {
- return ScmConfigKeys.OZONE_SCM_HTTP_BIND_PORT_DEFAULT;
- }
-
- @Override protected int getHttpsBindPortDefault() {
- return ScmConfigKeys.OZONE_SCM_HTTPS_BIND_PORT_DEFAULT;
- }
-
- @Override protected String getKeytabFile() {
- return ScmConfigKeys.OZONE_SCM_KEYTAB_FILE;
- }
-
- @Override protected String getSpnegoPrincipal() {
- return OzoneConfigKeys.OZONE_SCM_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL;
- }
-
- @Override protected String getEnabledKey() {
- return ScmConfigKeys.OZONE_SCM_HTTP_ENABLED_KEY;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
deleted file mode 100644
index 6bb3a22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/VersionInfo.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-/**
- * This is a class that tracks versions of SCM.
- */
-public final class VersionInfo {
-
- // We will just be normal and use positive counting numbers for versions.
- private final static VersionInfo[] VERSION_INFOS =
- {new VersionInfo("First version of SCM", 1)};
-
-
- public static final String DESCRIPTION_KEY = "Description";
- private final String description;
- private final int version;
-
- /**
- * Never created outside this class.
- *
- * @param description -- description
- * @param version -- version number
- */
- private VersionInfo(String description, int version) {
- this.description = description;
- this.version = version;
- }
-
- /**
- * Returns all versions.
- *
- * @return Version info array.
- */
- public static VersionInfo[] getAllVersions() {
- return VERSION_INFOS.clone();
- }
-
- /**
- * Returns the latest version.
- *
- * @return versionInfo
- */
- public static VersionInfo getLatestVersion() {
- return VERSION_INFOS[VERSION_INFOS.length - 1];
- }
-
- /**
- * Return description.
- *
- * @return String
- */
- public String getDescription() {
- return description;
- }
-
- /**
- * Return the version.
- *
- * @return int.
- */
- public int getVersion() {
- return version;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
deleted file mode 100644
index cfed7a8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManager.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- *
- * Block APIs.
- * Container is transparent to these APIs.
- */
-public interface BlockManager extends Closeable {
- /**
- * Allocates a new block for a given size.
- * @param size - Block Size
- * @param type Replication Type
- * @param factor - Replication Factor
- * @return AllocatedBlock
- * @throws IOException
- */
- AllocatedBlock allocateBlock(long size, OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, String owner) throws IOException;
-
- /**
- * Give the key to the block, get the pipeline info.
- * @param key - key to the block.
- * @return - Pipeline that used to access the block.
- * @throws IOException
- */
- Pipeline getBlock(String key) throws IOException;
-
- /**
- * Deletes a list of blocks in an atomic operation. Internally, SCM
- * writes these blocks into a {@link DeletedBlockLog} and deletes them
- * from SCM DB. If this is successful, given blocks are entering pending
- * deletion state and becomes invisible from SCM namespace.
- *
- * @param blockIDs block IDs. This is often the list of blocks of
- * a particular object key.
- * @throws IOException if exception happens, non of the blocks is deleted.
- */
- void deleteBlocks(List<String> blockIDs) throws IOException;
-
- /**
- * @return the block deletion transaction log maintained by SCM.
- */
- DeletedBlockLog getDeletedBlockLog();
-
- /**
- * Start block manager background services.
- * @throws IOException
- */
- void start() throws IOException;
-
- /**
- * Shutdown block manager background services.
- * @throws IOException
- */
- void stop() throws IOException;
-
- /**
- * @return the block deleting service executed in SCM.
- */
- SCMBlockDeletingService getSCMBlockDeletingService();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
deleted file mode 100644
index 5771080..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockManagerImpl.java
+++ /dev/null
@@ -1,530 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.UUID;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_DB;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .CHILL_MODE_EXCEPTION;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_FIND_BLOCK;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .INVALID_BLOCK_SIZE;
-
-/** Block Manager manages the block access for SCM. */
-public class BlockManagerImpl implements BlockManager, BlockmanagerMXBean {
- private static final Logger LOG =
- LoggerFactory.getLogger(BlockManagerImpl.class);
- // TODO : FIX ME : Hard coding the owner.
- // Currently only user of the block service is Ozone, CBlock manages blocks
- // by itself and does not rely on the Block service offered by SCM.
-
- private final NodeManager nodeManager;
- private final Mapping containerManager;
- private final MetadataStore blockStore;
-
- private final Lock lock;
- private final long containerSize;
- private final long cacheSize;
-
- private final DeletedBlockLog deletedBlockLog;
- private final SCMBlockDeletingService blockDeletingService;
-
- private final int containerProvisionBatchSize;
- private final Random rand;
- private ObjectName mxBean;
-
- /**
- * Constructor.
- *
- * @param conf - configuration.
- * @param nodeManager - node manager.
- * @param containerManager - container manager.
- * @param cacheSizeMB - cache size for level db store.
- * @throws IOException
- */
- public BlockManagerImpl(final Configuration conf,
- final NodeManager nodeManager, final Mapping containerManager,
- final int cacheSizeMB) throws IOException {
- this.nodeManager = nodeManager;
- this.containerManager = containerManager;
- this.cacheSize = cacheSizeMB;
-
- this.containerSize = OzoneConsts.GB * conf.getInt(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
- String scmMetaDataDir = metaDir.getPath();
-
- // Write the block key to container name mapping.
- File blockContainerDbPath = new File(scmMetaDataDir, BLOCK_DB);
- blockStore =
- MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(blockContainerDbPath)
- .setCacheSize(this.cacheSize * OzoneConsts.MB)
- .build();
-
- this.containerProvisionBatchSize =
- conf.getInt(
- ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE,
- ScmConfigKeys.OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT);
- rand = new Random();
- this.lock = new ReentrantLock();
-
- mxBean = MBeans.register("BlockManager", "BlockManagerImpl", this);
-
- // SCM block deleting transaction log and deleting service.
- deletedBlockLog = new DeletedBlockLogImpl(conf);
- long svcInterval =
- conf.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- long serviceTimeout =
- conf.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- blockDeletingService =
- new SCMBlockDeletingService(
- deletedBlockLog, containerManager, nodeManager, svcInterval,
- serviceTimeout, conf);
- }
-
- /**
- * Start block manager services.
- *
- * @throws IOException
- */
- public void start() throws IOException {
- this.blockDeletingService.start();
- }
-
- /**
- * Shutdown block manager services.
- *
- * @throws IOException
- */
- public void stop() throws IOException {
- this.blockDeletingService.shutdown();
- this.close();
- }
-
- /**
- * Pre allocate specified count of containers for block creation.
- *
- * @param count - Number of containers to allocate.
- * @param type - Type of containers
- * @param factor - how many copies needed for this container.
- * @throws IOException
- */
- private void preAllocateContainers(int count, ReplicationType type,
- ReplicationFactor factor, String owner)
- throws IOException {
- lock.lock();
- try {
- for (int i = 0; i < count; i++) {
- String containerName = UUID.randomUUID().toString();
- ContainerInfo containerInfo = null;
- try {
- // TODO: Fix this later when Ratis is made the Default.
- containerInfo = containerManager.allocateContainer(type, factor,
- containerName, owner);
-
- if (containerInfo == null) {
- LOG.warn("Unable to allocate container.");
- continue;
- }
- } catch (IOException ex) {
- LOG.warn("Unable to allocate container: {}", ex);
- continue;
- }
- }
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Allocates a block in a container and returns that info.
- *
- * @param size - Block Size
- * @param type Replication Type
- * @param factor - Replication Factor
- * @return Allocated block
- * @throws IOException on failure.
- */
- @Override
- public AllocatedBlock allocateBlock(final long size,
- ReplicationType type, ReplicationFactor factor, String owner)
- throws IOException {
- LOG.trace("Size;{} , type : {}, factor : {} ", size, type, factor);
-
- if (size < 0 || size > containerSize) {
- LOG.warn("Invalid block size requested : {}", size);
- throw new SCMException("Unsupported block size: " + size,
- INVALID_BLOCK_SIZE);
- }
-
- if (!nodeManager.isOutOfChillMode()) {
- LOG.warn("Not out of Chill mode.");
- throw new SCMException("Unable to create block while in chill mode",
- CHILL_MODE_EXCEPTION);
- }
-
- lock.lock();
- try {
- /*
- Here is the high level logic.
-
- 1. First we check if there are containers in ALLOCATED state,
- that is
- SCM has allocated them in the SCM namespace but the
- corresponding
- container has not been created in the Datanode yet. If we
- have any
- in that state, we will return that to the client, which allows
- client to finish creating those containers. This is a sort of
- greedy
- algorithm, our primary purpose is to get as many containers as
- possible.
-
- 2. If there are no allocated containers -- Then we find a Open
- container that matches that pattern.
-
- 3. If both of them fail, the we will pre-allocate a bunch of
- conatainers in SCM and try again.
-
- TODO : Support random picking of two containers from the list.
- So we
- can use different kind of policies.
- */
-
- ContainerInfo containerInfo;
-
- // Look for ALLOCATED container that matches all other parameters.
- containerInfo =
- containerManager
- .getStateManager()
- .getMatchingContainer(
- size, owner, type, factor, OzoneProtos.LifeCycleState
- .ALLOCATED);
- if (containerInfo != null) {
- containerManager.updateContainerState(containerInfo.getContainerName(),
- OzoneProtos.LifeCycleEvent.CREATE);
- return newBlock(containerInfo, OzoneProtos.LifeCycleState.ALLOCATED);
- }
-
- // Since we found no allocated containers that match our criteria, let us
- // look for OPEN containers that match the criteria.
- containerInfo =
- containerManager
- .getStateManager()
- .getMatchingContainer(size, owner, type, factor, OzoneProtos
- .LifeCycleState.OPEN);
- if (containerInfo != null) {
- return newBlock(containerInfo, OzoneProtos.LifeCycleState.OPEN);
- }
-
- // We found neither ALLOCATED or OPEN Containers. This generally means
- // that most of our containers are full or we have not allocated
- // containers of the type and replication factor. So let us go and
- // allocate some.
- preAllocateContainers(containerProvisionBatchSize, type, factor, owner);
-
- // Since we just allocated a set of containers this should work
- containerInfo =
- containerManager
- .getStateManager()
- .getMatchingContainer(
- size, owner, type, factor, OzoneProtos.LifeCycleState
- .ALLOCATED);
- if (containerInfo != null) {
- containerManager.updateContainerState(containerInfo.getContainerName(),
- OzoneProtos.LifeCycleEvent.CREATE);
- return newBlock(containerInfo, OzoneProtos.LifeCycleState.ALLOCATED);
- }
-
- // we have tried all strategies we know and but somehow we are not able
- // to get a container for this block. Log that info and return a null.
- LOG.error(
- "Unable to allocate a block for the size: {}, type: {}, " +
- "factor: {}",
- size,
- type,
- factor);
- return null;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * newBlock - returns a new block assigned to a container.
- *
- * @param containerInfo - Container Info.
- * @param state - Current state of the container.
- * @return AllocatedBlock
- */
- private AllocatedBlock newBlock(
- ContainerInfo containerInfo, OzoneProtos.LifeCycleState state)
- throws IOException {
-
- // TODO : Replace this with Block ID.
- String blockKey = UUID.randomUUID().toString();
- boolean createContainer = (state == OzoneProtos.LifeCycleState.ALLOCATED);
-
- AllocatedBlock.Builder abb =
- new AllocatedBlock.Builder()
- .setKey(blockKey)
- // TODO : Use containerinfo instead of pipeline.
- .setPipeline(containerInfo.getPipeline())
- .setShouldCreateContainer(createContainer);
- LOG.trace("New block allocated : {} Container ID: {}", blockKey,
- containerInfo.toString());
-
- if (containerInfo.getPipeline().getMachines().size() == 0) {
- LOG.error("Pipeline Machine count is zero.");
- return null;
- }
-
- // Persist this block info to the blockStore DB, so getBlock(key) can
- // find which container the block lives.
- // TODO : Remove this DB in future
- // and make this a KSM operation. Category: SCALABILITY.
- if (containerInfo.getPipeline().getMachines().size() > 0) {
- blockStore.put(
- DFSUtil.string2Bytes(blockKey),
- DFSUtil.string2Bytes(containerInfo.getPipeline().getContainerName()));
- }
- return abb.build();
- }
-
- /**
- * Given a block key, return the Pipeline information.
- *
- * @param key - block key assigned by SCM.
- * @return Pipeline (list of DNs and leader) to access the block.
- * @throws IOException
- */
- @Override
- public Pipeline getBlock(final String key) throws IOException {
- lock.lock();
- try {
- byte[] containerBytes = blockStore.get(DFSUtil.string2Bytes(key));
- if (containerBytes == null) {
- throw new SCMException(
- "Specified block key does not exist. key : " + key,
- FAILED_TO_FIND_BLOCK);
- }
-
- String containerName = DFSUtil.bytes2String(containerBytes);
- ContainerInfo containerInfo = containerManager.getContainer(
- containerName);
- if (containerInfo == null) {
- LOG.debug("Container {} allocated by block service"
- + "can't be found in SCM", containerName);
- throw new SCMException(
- "Unable to find container for the block",
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
- return containerInfo.getPipeline();
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Deletes a list of blocks in an atomic operation. Internally, SCM writes
- * these blocks into a
- * {@link DeletedBlockLog} and deletes them from SCM DB. If this is
- * successful, given blocks are
- * entering pending deletion state and becomes invisible from SCM namespace.
- *
- * @param blockIDs block IDs. This is often the list of blocks of a
- * particular object key.
- * @throws IOException if exception happens, non of the blocks is deleted.
- */
- @Override
- public void deleteBlocks(List<String> blockIDs) throws IOException {
- if (!nodeManager.isOutOfChillMode()) {
- throw new SCMException("Unable to delete block while in chill mode",
- CHILL_MODE_EXCEPTION);
- }
-
- lock.lock();
- LOG.info("Deleting blocks {}", String.join(",", blockIDs));
- Map<String, List<String>> containerBlocks = new HashMap<>();
- BatchOperation batch = new BatchOperation();
- BatchOperation rollbackBatch = new BatchOperation();
- // TODO: track the block size info so that we can reclaim the container
- // TODO: used space when the block is deleted.
- try {
- for (String blockKey : blockIDs) {
- byte[] blockKeyBytes = DFSUtil.string2Bytes(blockKey);
- byte[] containerBytes = blockStore.get(blockKeyBytes);
- if (containerBytes == null) {
- throw new SCMException(
- "Specified block key does not exist. key : " + blockKey,
- FAILED_TO_FIND_BLOCK);
- }
- batch.delete(blockKeyBytes);
- rollbackBatch.put(blockKeyBytes, containerBytes);
-
- // Merge blocks to a container to blocks mapping,
- // prepare to persist this info to the deletedBlocksLog.
- String containerName = DFSUtil.bytes2String(containerBytes);
- if (containerBlocks.containsKey(containerName)) {
- containerBlocks.get(containerName).add(blockKey);
- } else {
- List<String> item = new ArrayList<>();
- item.add(blockKey);
- containerBlocks.put(containerName, item);
- }
- }
-
- // We update SCM DB first, so if this step fails, we end up here,
- // nothing gets into the delLog so no blocks will be accidentally
- // removed. If we write the log first, once log is written, the
- // async deleting service will start to scan and might be picking
- // up some blocks to do real deletions, that might cause data loss.
- blockStore.writeBatch(batch);
- try {
- deletedBlockLog.addTransactions(containerBlocks);
- } catch (IOException e) {
- try {
- // If delLog update is failed, we need to rollback the changes.
- blockStore.writeBatch(rollbackBatch);
- } catch (IOException rollbackException) {
- // This is a corner case. AddTX fails and rollback also fails,
- // this will leave these blocks in inconsistent state. They were
- // moved to pending deletion state in SCM DB but were not written
- // into delLog so real deletions would not be done. Blocks become
- // to be invisible from namespace but actual data are not removed.
- // We log an error here so admin can manually check and fix such
- // errors.
- LOG.error(
- "Blocks might be in inconsistent state because"
- + " they were moved to pending deletion state in SCM DB but"
- + " not written into delLog. Admin can manually add them"
- + " into delLog for deletions. Inconsistent block list: {}",
- String.join(",", blockIDs),
- e);
- throw rollbackException;
- }
- throw new IOException(
- "Skip writing the deleted blocks info to"
- + " the delLog because addTransaction fails. Batch skipped: "
- + String.join(",", blockIDs),
- e);
- }
- // TODO: Container report handling of the deleted blocks:
- // Remove tombstone and update open container usage.
- // We will revisit this when the closed container replication is done.
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public DeletedBlockLog getDeletedBlockLog() {
- return this.deletedBlockLog;
- }
-
- @VisibleForTesting
- public String getDeletedKeyName(String key) {
- return StringUtils.format(".Deleted/%s", key);
- }
-
- /**
- * Close the resources for BlockManager.
- *
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- if (blockStore != null) {
- blockStore.close();
- }
- if (deletedBlockLog != null) {
- deletedBlockLog.close();
- }
- blockDeletingService.shutdown();
- if (mxBean != null) {
- MBeans.unregister(mxBean);
- mxBean = null;
- }
- }
-
- @Override
- public int getOpenContainersNo() {
- return 0;
- // TODO : FIX ME : The open container being a single number does not make
- // sense.
- // We have to get open containers by Replication Type and Replication
- // factor. Hence returning 0 for now.
- // containers.get(OzoneProtos.LifeCycleState.OPEN).size();
- }
-
- @Override
- public SCMBlockDeletingService getSCMBlockDeletingService() {
- return this.blockDeletingService;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
deleted file mode 100644
index efcfc63..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/BlockmanagerMXBean.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.block;
-
-
-/**
- * JMX interface for the block manager.
- */
-public interface BlockmanagerMXBean {
-
- /**
- * Number of open containers manager by the block manager.
- */
- int getOpenContainersNo();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
deleted file mode 100644
index 31c61dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DatanodeDeletedBlockTransactions.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-import org.apache.hadoop.ozone.scm.container.Mapping;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-
-import com.google.common.collect.ArrayListMultimap;
-
-/**
- * A wrapper class to hold info about datanode and all deleted block
- * transactions that will be sent to this datanode.
- */
-public class DatanodeDeletedBlockTransactions {
- private int nodeNum;
- // The throttle size for each datanode.
- private int maximumAllowedTXNum;
- // Current counter of inserted TX.
- private int currentTXNum;
- private Mapping mappingService;
- // A list of TXs mapped to a certain datanode ID.
- private final ArrayListMultimap<DatanodeID, DeletedBlocksTransaction>
- transactions;
-
- DatanodeDeletedBlockTransactions(Mapping mappingService,
- int maximumAllowedTXNum, int nodeNum) {
- this.transactions = ArrayListMultimap.create();
- this.mappingService = mappingService;
- this.maximumAllowedTXNum = maximumAllowedTXNum;
- this.nodeNum = nodeNum;
- }
-
- public void addTransaction(DeletedBlocksTransaction tx) throws IOException {
- ContainerInfo info = null;
- try {
- info = mappingService.getContainer(tx.getContainerName());
- } catch (IOException e) {
- SCMBlockDeletingService.LOG.warn("Got container info error.", e);
- }
-
- if (info == null) {
- SCMBlockDeletingService.LOG.warn(
- "Container {} not found, continue to process next",
- tx.getContainerName());
- return;
- }
-
- for (DatanodeID dnID : info.getPipeline().getMachines()) {
- if (transactions.containsKey(dnID)) {
- List<DeletedBlocksTransaction> txs = transactions.get(dnID);
- if (txs != null && txs.size() < maximumAllowedTXNum) {
- boolean hasContained = false;
- for (DeletedBlocksTransaction t : txs) {
- if (t.getContainerName().equals(tx.getContainerName())) {
- hasContained = true;
- break;
- }
- }
-
- if (!hasContained) {
- txs.add(tx);
- currentTXNum++;
- }
- }
- } else {
- currentTXNum++;
- transactions.put(dnID, tx);
- }
- SCMBlockDeletingService.LOG.debug("Transaction added: {} <- TX({})", dnID,
- tx.getTxID());
- }
- }
-
- Set<DatanodeID> getDatanodes() {
- return transactions.keySet();
- }
-
- boolean isEmpty() {
- return transactions.isEmpty();
- }
-
- boolean hasTransactions(DatanodeID dnID) {
- return transactions.containsKey(dnID) && !transactions.get(dnID).isEmpty();
- }
-
- List<DeletedBlocksTransaction> getDatanodeTransactions(
- DatanodeID dnID) {
- return transactions.get(dnID);
- }
-
- List<String> getTransactionIDList(DatanodeID dnID) {
- if (hasTransactions(dnID)) {
- return transactions.get(dnID).stream()
- .map(DeletedBlocksTransaction::getTxID).map(String::valueOf)
- .collect(Collectors.toList());
- } else {
- return Collections.emptyList();
- }
- }
-
- boolean isFull() {
- return currentTXNum >= maximumAllowedTXNum * nodeNum;
- }
-
- int getTXNum() {
- return currentTXNum;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
deleted file mode 100644
index d8af853..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/block/DeletedBlockLog.java
+++ /dev/null
@@ -1,124 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.block;
-
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The DeletedBlockLog is a persisted log in SCM to keep tracking
- * container blocks which are under deletion. It maintains info
- * about under-deletion container blocks that notified by KSM,
- * and the state how it is processed.
- */
-public interface DeletedBlockLog extends Closeable {
-
- /**
- * A limit size list of transactions. Note count is the max number
- * of TXs to return, we might not be able to always return this
- * number. and the processCount of those transactions
- * should be [0, MAX_RETRY).
- *
- * @param count - number of transactions.
- * @return a list of BlockDeletionTransaction.
- */
- List<DeletedBlocksTransaction> getTransactions(int count)
- throws IOException;
-
- /**
- * Scan entire log once and returns TXs to DatanodeDeletedBlockTransactions.
- * Once DatanodeDeletedBlockTransactions is full, the scan behavior will
- * stop.
- * @param transactions a list of TXs will be set into.
- * @throws IOException
- */
- void getTransactions(DatanodeDeletedBlockTransactions transactions)
- throws IOException;
-
- /**
- * Return all failed transactions in the log. A transaction is considered
- * to be failed if it has been sent more than MAX_RETRY limit and its
- * count is reset to -1.
- *
- * @return a list of failed deleted block transactions.
- * @throws IOException
- */
- List<DeletedBlocksTransaction> getFailedTransactions()
- throws IOException;
-
- /**
- * Increments count for given list of transactions by 1.
- * The log maintains a valid range of counts for each transaction
- * [0, MAX_RETRY]. If exceed this range, resets it to -1 to indicate
- * the transaction is no longer valid.
- *
- * @param txIDs - transaction ID.
- */
- void incrementCount(List<Long> txIDs)
- throws IOException;
-
- /**
- * Commits a transaction means to delete all footprints of a transaction
- * from the log. This method doesn't guarantee all transactions can be
- * successfully deleted, it tolerate failures and tries best efforts to.
- *
- * @param txIDs - transaction IDs.
- */
- void commitTransactions(List<Long> txIDs) throws IOException;
-
- /**
- * Creates a block deletion transaction and adds that into the log.
- *
- * @param containerName - container name.
- * @param blocks - blocks that belong to the same container.
- *
- * @throws IOException
- */
- void addTransaction(String containerName, List<String> blocks)
- throws IOException;
-
- /**
- * Creates block deletion transactions for a set of containers,
- * add into the log and persist them atomically. An object key
- * might be stored in multiple containers and multiple blocks,
- * this API ensures that these updates are done in atomic manner
- * so if any of them fails, the entire operation fails without
- * any updates to the log. Note, this doesn't mean to create only
- * one transaction, it creates multiple transactions (depends on the
- * number of containers) together (on success) or non (on failure).
- *
- * @param containerBlocksMap a map of containerBlocks.
- * @throws IOException
- */
- void addTransactions(Map<String, List<String>> containerBlocksMap)
- throws IOException;
-
- /**
- * Returns the total number of valid transactions. A transaction is
- * considered to be valid as long as its count is in range [0, MAX_RETRY].
- *
- * @return number of a valid transactions.
- * @throws IOException
- */
- int getNumOfValidTransactions() throws IOException;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
new file mode 100644
index 0000000..90fe802
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServer.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * This class tests the basics of CBlock server. Mainly about the four
+ * operations on volumes: create, delete, info and list.
+ */
+public class TestCBlockServer {
+ private static CBlockManager cBlockManager;
+ private static OzoneConfiguration conf;
+
+ @Before
+ public void setup() throws Exception {
+ ScmClient storageClient = new MockStorageClient();
+ conf = new OzoneConfiguration();
+ conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+ cBlockManager = new CBlockManager(conf, storageClient);
+ cBlockManager.start();
+ }
+
+ @After
+ public void clean() {
+ cBlockManager.stop();
+ cBlockManager.join();
+ cBlockManager.clean();
+ }
+
+ /**
+ * Test create volume for different users.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateVolume() throws Exception {
+ String userName1 = "user" + RandomStringUtils.randomNumeric(5);
+ String userName2 = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5);
+ String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 1L*1024*1024;
+ int blockSize = 4096;
+ cBlockManager.createVolume(userName1, volumeName1, volumeSize, blockSize);
+ List<VolumeInfo> volumes = cBlockManager.listVolume(userName1);
+ assertEquals(1, volumes.size());
+ VolumeInfo existingVolume = volumes.get(0);
+ assertEquals(userName1, existingVolume.getUserName());
+ assertEquals(volumeName1, existingVolume.getVolumeName());
+ assertEquals(volumeSize, existingVolume.getVolumeSize());
+ assertEquals(blockSize, existingVolume.getBlockSize());
+
+ cBlockManager.createVolume(userName1, volumeName2, volumeSize, blockSize);
+ cBlockManager.createVolume(userName2, volumeName1, volumeSize, blockSize);
+ volumes = cBlockManager.listVolume(userName1);
+ assertEquals(2, volumes.size());
+ volumes = cBlockManager.listVolume(userName2);
+ assertEquals(1, volumes.size());
+ }
+
+ /**
+ * Test delete volume.
+ * @throws Exception
+ */
+ @Test
+ public void testDeleteVolume() throws Exception {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName1 = "volume" + RandomStringUtils.randomNumeric(5);
+ String volumeName2 = "volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 1L*1024*1024;
+ int blockSize = 4096;
+ cBlockManager.createVolume(userName, volumeName1, volumeSize, blockSize);
+ cBlockManager.createVolume(userName, volumeName2, volumeSize, blockSize);
+ cBlockManager.deleteVolume(userName, volumeName1, true);
+ List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
+ assertEquals(1, volumes.size());
+
+ VolumeInfo existingVolume = volumes.get(0);
+ assertEquals(userName, existingVolume.getUserName());
+ assertEquals(volumeName2, existingVolume.getVolumeName());
+ assertEquals(volumeSize, existingVolume.getVolumeSize());
+ assertEquals(blockSize, existingVolume.getBlockSize());
+ }
+
+ /**
+ * Test info volume.
+ *
+ * TODO : usage field is not being tested (as it is not implemented yet)
+ * @throws Exception
+ */
+ @Test
+ public void testInfoVolume() throws Exception {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 1L*1024*1024;
+ int blockSize = 4096;
+ cBlockManager.createVolume(userName, volumeName, volumeSize, blockSize);
+ VolumeInfo info = cBlockManager.infoVolume(userName, volumeName);
+ assertEquals(userName, info.getUserName());
+ assertEquals(volumeName, info.getVolumeName());
+ assertEquals(volumeSize, info.getVolumeSize());
+ assertEquals(blockSize, info.getBlockSize());
+ }
+
+ /**
+ * Test listing a number of volumes.
+ * @throws Exception
+ */
+ @Test
+ public void testListVolume() throws Exception {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName ="volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 1L*1024*1024;
+ int blockSize = 4096;
+ int volumeNum = 100;
+ for (int i = 0; i<volumeNum; i++) {
+ cBlockManager.createVolume(userName, volumeName + i,
+ volumeSize, blockSize);
+ }
+ List<VolumeInfo> volumes = cBlockManager.listVolume(userName);
+ assertEquals(volumeNum, volumes.size());
+ Set<String> volumeIds = new HashSet<>();
+ for (int i = 0; i<volumeNum; i++) {
+ VolumeInfo volumeInfo = volumes.get(i);
+ assertEquals(userName, volumeInfo.getUserName());
+ assertFalse(volumeIds.contains(volumeName + i));
+ volumeIds.add(volumeName + i);
+ assertEquals(volumeSize, volumeInfo.getVolumeSize());
+ assertEquals(blockSize, volumeInfo.getBlockSize());
+ }
+ for (int i = 0; i<volumeNum; i++) {
+ assertTrue(volumeIds.contains(volumeName + i));
+ }
+ }
+
+ /**
+ * Test listing a number of volumes.
+ * @throws Exception
+ */
+ @Test
+ public void testListVolumes() throws Exception {
+ String volumeName ="volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 1L*1024*1024;
+ int blockSize = 4096;
+ int volumeNum = 100;
+ int userCount = 10;
+
+ assertTrue("We need at least one volume for each user",
+ userCount < volumeNum);
+
+ for (int i = 0; i<volumeNum; i++) {
+ String userName =
+ "user-" + (i % userCount);
+ cBlockManager.createVolume(userName, volumeName + i,
+ volumeSize, blockSize);
+ }
+ List<VolumeInfo> allVolumes = cBlockManager.listVolumes();
+ //check if we have the volumes from all the users.
+
+ Set<String> volumeIds = new HashSet<>();
+ Set<String> usernames = new HashSet<>();
+ for (int i = 0; i < allVolumes.size(); i++) {
+ VolumeInfo volumeInfo = allVolumes.get(i);
+ assertFalse(volumeIds.contains(volumeName + i));
+ usernames.add(volumeInfo.getUserName());
+ volumeIds.add(volumeName + i);
+ assertEquals(volumeSize, volumeInfo.getVolumeSize());
+ assertEquals(blockSize, volumeInfo.getBlockSize());
+ }
+
+ assertEquals(volumeNum, volumeIds.size());
+ for (int i = 0; i<volumeNum; i++) {
+ assertTrue(volumeIds.contains(volumeName + i));
+ }
+
+ assertEquals(userCount, usernames.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
new file mode 100644
index 0000000..1ea183b
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockServerPersistence.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * Test the CBlock server state is maintained in persistent storage and can be
+ * recovered on CBlock server restart.
+ */
+public class TestCBlockServerPersistence {
+
+ /**
+ * Test when cblock server fails with volume meta data, the meta data can be
+ * restored correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testWriteToPersistentStore() throws Exception {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ String userName = "testWriteToPersistentStore";
+ String volumeName1 = "testVolume1";
+ String volumeName2 = "testVolume2";
+ long volumeSize1 = 30L*1024*1024*1024;
+ long volumeSize2 = 15L*1024*1024*1024;
+ int blockSize = 4096;
+ CBlockManager cBlockManager = null;
+ CBlockManager cBlockManager1 = null;
+ String path = GenericTestUtils
+ .getTempPath(TestCBlockServerPersistence.class.getSimpleName());
+ File filePath = new File(path);
+ if(!filePath.exists() && !filePath.mkdirs()) {
+ throw new IOException("Unable to create test DB dir");
+ }
+ conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+ "/testCblockPersistence.dat"));
+ try {
+ ScmClient storageClient = new MockStorageClient();
+ conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+ cBlockManager = new CBlockManager(conf, storageClient);
+ cBlockManager.start();
+ cBlockManager.createVolume(userName, volumeName1, volumeSize1, blockSize);
+ cBlockManager.createVolume(userName, volumeName2, volumeSize2, blockSize);
+ List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes();
+ // close the cblock server. Since meta data is written to disk on volume
+ // creation, closing server here is the same as a cblock server crash.
+ cBlockManager.close();
+ cBlockManager.stop();
+ cBlockManager.join();
+ cBlockManager = null;
+ assertEquals(2, allVolumes.size());
+ VolumeDescriptor volumeDescriptor1 = allVolumes.get(0);
+ VolumeDescriptor volumeDescriptor2 = allVolumes.get(1);
+
+ // create a new cblock server instance. This is just the
+ // same as restarting cblock server.
+ ScmClient storageClient1 = new MockStorageClient();
+ OzoneConfiguration conf1 = new OzoneConfiguration();
+ conf1.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+ "/testCblockPersistence.dat"));
+ conf1.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf1.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+ cBlockManager1 = new CBlockManager(conf1, storageClient1);
+ cBlockManager1.start();
+ List<VolumeDescriptor> allVolumes1 = cBlockManager1.getAllVolumes();
+ assertEquals(2, allVolumes1.size());
+ VolumeDescriptor newvolumeDescriptor1 = allVolumes1.get(0);
+ VolumeDescriptor newvolumeDescriptor2 = allVolumes1.get(1);
+
+ // It seems levelDB iterator gets keys in the same order as keys
+ // are inserted, in which case the else clause should never happen.
+ // But still kept the second clause if it is possible to get different
+ // key ordering from leveldb. And we do not rely on the ordering of keys
+ // here.
+ if (volumeDescriptor1.getVolumeName().equals(
+ newvolumeDescriptor1.getVolumeName())) {
+ assertEquals(volumeDescriptor1.toString(),
+ newvolumeDescriptor1.toString());
+ assertEquals(volumeDescriptor2.toString(),
+ newvolumeDescriptor2.toString());
+ } else {
+ assertEquals(volumeDescriptor1.toString(),
+ newvolumeDescriptor2.toString());
+ assertEquals(volumeDescriptor2.toString(),
+ newvolumeDescriptor1.toString());
+ }
+ } finally {
+ if (cBlockManager != null) {
+ cBlockManager.clean();
+ }
+ if (cBlockManager1 != null) {
+ cBlockManager1.close();
+ cBlockManager1.stop();
+ cBlockManager1.join();
+ cBlockManager1.clean();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
new file mode 100644
index 0000000..6eb7ea6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestLocalBlockCache.java
@@ -0,0 +1,444 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockIStorageImpl;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Time;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static java.lang.Math.abs;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_TRACE_IO;
+
+/**
+ * Tests for local cache.
+ */
+public class TestLocalBlockCache {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLocalBlockCache.class);
+ private final static long GB = 1024 * 1024 * 1024;
+ private final static int KB = 1024;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration config;
+ private static StorageContainerLocationProtocolClientSideTranslatorPB
+ storageContainerLocationClient;
+ private static XceiverClientManager xceiverClientManager;
+
+ @BeforeClass
+ public static void init() throws IOException {
+ config = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestLocalBlockCache.class.getSimpleName());
+ config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ cluster = new MiniOzoneClassicCluster.Builder(config)
+ .numDataNodes(1)
+ .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+ storageContainerLocationClient = cluster
+ .createStorageContainerLocationClient();
+ xceiverClientManager = new XceiverClientManager(config);
+ }
+
+ @AfterClass
+ public static void shutdown() throws InterruptedException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+ }
+
+ /**
+ * getContainerPipelines creates a set of containers and returns the
+ * Pipelines that define those containers.
+ *
+ * @param count - Number of containers to create.
+ * @return - List of Pipelines.
+ * @throws IOException throws Exception
+ */
+ private List<Pipeline> getContainerPipeline(int count) throws IOException {
+ List<Pipeline> containerPipelines = new LinkedList<>();
+ for (int x = 0; x < count; x++) {
+ String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+ String containerName = "container" + RandomStringUtils.randomNumeric(10);
+ Pipeline pipeline =
+ storageContainerLocationClient.allocateContainer(
+ xceiverClientManager.getType(),
+ xceiverClientManager.getFactor(), containerName, "CBLOCK");
+ XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+ ContainerProtocolCalls.createContainer(client, traceID);
+ // This step is needed since we set private data on pipelines, when we
+ // read the list from CBlockServer. So we mimic that action here.
+ pipeline.setData(Longs.toByteArray(x));
+ containerPipelines.add(pipeline);
+ xceiverClientManager.releaseClient(client);
+ }
+ return containerPipelines;
+ }
+
+ /**
+ * This test creates a cache and performs a simple write / read.
+ * Due to the cache - we have Read-after-write consistency for cBlocks.
+ *
+ * @throws IOException throws Exception
+ */
+ @Test
+ public void testCacheWriteRead() throws IOException,
+ InterruptedException, TimeoutException {
+ final long blockID = 0;
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ String dataHash = DigestUtils.sha256Hex(data);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(1, metrics.getNumWriteOps());
+ // Please note that this read is from the local cache.
+ LogicalBlock block = cache.get(blockID);
+ Assert.assertEquals(1, metrics.getNumReadOps());
+ Assert.assertEquals(1, metrics.getNumReadCacheHits());
+ Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+
+ cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(2, metrics.getNumWriteOps());
+ // Please note that this read is from the local cache.
+ block = cache.get(blockID + 1);
+ Assert.assertEquals(2, metrics.getNumReadOps());
+ Assert.assertEquals(2, metrics.getNumReadCacheHits());
+ Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+ String readHash = DigestUtils.sha256Hex(block.getData().array());
+ Assert.assertEquals("File content does not match.", dataHash, readHash);
+ GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+ cache.close();
+
+ }
+
+ @Test
+ public void testCacheWriteToRemoteContainer() throws IOException,
+ InterruptedException, TimeoutException {
+ final long blockID = 0;
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+ GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+ cache.close();
+ }
+
+ @Test
+ public void testCacheWriteToRemote50KBlocks() throws IOException,
+ InterruptedException, TimeoutException {
+ final long totalBlocks = 50 * 1000;
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * 1024)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ long startTime = Time.monotonicNow();
+ for (long blockid = 0; blockid < totalBlocks; blockid++) {
+ cache.put(blockid, data.getBytes(StandardCharsets.UTF_8));
+ }
+ Assert.assertEquals(totalBlocks, metrics.getNumWriteOps());
+ Assert.assertEquals(totalBlocks, metrics.getNumBlockBufferUpdates());
+ LOG.info("Wrote 50K blocks, waiting for replication to finish.");
+ GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+ long endTime = Time.monotonicNow();
+ LOG.info("Time taken for writing {} blocks is {} seconds", totalBlocks,
+ TimeUnit.MILLISECONDS.toSeconds(endTime - startTime));
+ // TODO: Read this data back.
+ cache.close();
+ }
+
+ @Test
+ public void testCacheInvalidBlock() throws IOException {
+ final int blockID = 1024;
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ // Read a non-existent block ID.
+ LogicalBlock block = cache.get(blockID);
+ Assert.assertNotNull(block);
+ Assert.assertEquals(4 * 1024, block.getData().array().length);
+ Assert.assertEquals(1, metrics.getNumReadOps());
+ Assert.assertEquals(1, metrics.getNumReadLostBlocks());
+ Assert.assertEquals(1, metrics.getNumReadCacheMiss());
+ cache.close();
+ }
+
+ @Test
+ public void testReadWriteCorrectness() throws IOException,
+ InterruptedException, TimeoutException {
+ Random r = new Random();
+ final int maxBlock = 12500000;
+ final int blockCount = 10 * 1000;
+ Map<Long, String> blockShaMap = new HashMap<>();
+ List<Pipeline> pipelines = getContainerPipeline(10);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ final CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ for (int x = 0; x < blockCount; x++) {
+ String data = RandomStringUtils.random(4 * 1024);
+ String dataHash = DigestUtils.sha256Hex(data);
+ long blockId = abs(r.nextInt(maxBlock));
+ blockShaMap.put(blockId, dataHash);
+ cache.put(blockId, data.getBytes(StandardCharsets.UTF_8));
+ }
+ Assert.assertEquals(blockCount, metrics.getNumWriteOps());
+ GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+ LOG.info("Finished with putting blocks ..starting reading blocks back. " +
+ "unique blocks : {}", blockShaMap.size());
+ // Test reading from local cache.
+ for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
+ LogicalBlock block = cache.get(entry.getKey());
+ String blockSha = DigestUtils.sha256Hex(block.getData().array());
+ Assert.assertEquals("Block data is not equal", entry.getValue(),
+ blockSha);
+ }
+ Assert.assertEquals(blockShaMap.size(), metrics.getNumReadOps());
+ Assert.assertEquals(blockShaMap.size(), metrics.getNumReadCacheHits());
+ Assert.assertEquals(0, metrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+
+ LOG.info("Finished with reading blocks, SUCCESS.");
+ // Close and discard local cache.
+ cache.close();
+ LOG.info("Closing the and destroying local cache");
+ CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher newflusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, newMetrics);
+ Assert.assertEquals(0, newMetrics.getNumReadCacheHits());
+ CBlockLocalCache newCache = null;
+ try {
+ newCache = CBlockLocalCache.newBuilder()
+ .setConfiguration(this.config)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(newflusher)
+ .setCBlockTargetMetrics(newMetrics)
+ .build();
+ newCache.start();
+ for (Map.Entry<Long, String> entry : blockShaMap.entrySet()) {
+ LogicalBlock block = newCache.get(entry.getKey());
+ String blockSha = DigestUtils.sha256Hex(block.getData().array());
+ Assert.assertEquals("Block data is not equal", entry.getValue(),
+ blockSha);
+ }
+
+ Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadOps());
+ Assert.assertEquals(blockShaMap.size(), newMetrics.getNumReadCacheHits());
+ Assert.assertEquals(0, newMetrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
+
+ LOG.info("Finished with reading blocks from remote cache, SUCCESS.");
+ } finally {
+ if (newCache != null) {
+ newCache.close();
+ }
+ }
+ }
+
+ @Test
+ public void testStorageImplReadWrite() throws IOException,
+ InterruptedException, TimeoutException {
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 50L * (1024L * 1024L * 1024L);
+ int blockSize = 4096;
+ byte[] data =
+ RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
+ .getBytes(StandardCharsets.UTF_8);
+ String hash = DigestUtils.sha256Hex(data);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(this.config,
+ xceiverClientManager, metrics);
+ CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+ .setUserName(userName)
+ .setVolumeName(volumeName)
+ .setVolumeSize(volumeSize)
+ .setBlockSize(blockSize)
+ .setContainerList(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setConf(this.config)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ ozoneStore.write(data, 0);
+
+ byte[] newData = new byte[10 * 1024 * 1024];
+ ozoneStore.read(newData, 0);
+ String newHash = DigestUtils.sha256Hex(newData);
+ Assert.assertEquals("hashes don't match.", hash, newHash);
+ GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
+ 100, 20 * 1000);
+ ozoneStore.close();
+ }
+
+ //@Test
+ // Disabling this test for time being since the bug in JSCSI
+ // forces us always to have a local cache.
+ public void testStorageImplNoLocalCache() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration oConfig = new OzoneConfiguration();
+ oConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+ oConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ String userName = "user" + RandomStringUtils.randomNumeric(5);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(5);
+ long volumeSize = 50L * (1024L * 1024L * 1024L);
+ int blockSize = 4096;
+ byte[] data =
+ RandomStringUtils.randomAlphanumeric(10 * (1024 * 1024))
+ .getBytes(StandardCharsets.UTF_8);
+ String hash = DigestUtils.sha256Hex(data);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(oConfig,
+ xceiverClientManager, metrics);
+ CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+ .setUserName(userName)
+ .setVolumeName(volumeName)
+ .setVolumeSize(volumeSize)
+ .setBlockSize(blockSize)
+ .setContainerList(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setConf(oConfig)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ ozoneStore.write(data, 0);
+
+ byte[] newData = new byte[10 * 1024 * 1024];
+ ozoneStore.read(newData, 0);
+ String newHash = DigestUtils.sha256Hex(newData);
+ Assert.assertEquals("hashes don't match.", hash, newHash);
+ GenericTestUtils.waitFor(() -> !ozoneStore.getCache().isDirtyCache(),
+ 100, 20 * 1000);
+ ozoneStore.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
new file mode 100644
index 0000000..8d1a865
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/kubernetes/TestDynamicProvisioner.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.kubernetes;
+
+import io.kubernetes.client.JSON;
+import io.kubernetes.client.models.V1PersistentVolume;
+import io.kubernetes.client.models.V1PersistentVolumeClaim;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+
+/**
+ * Test the resource generation of Dynamic Provisioner.
+ */
+public class TestDynamicProvisioner {
+
+ @Test
+ public void persitenceVolumeBuilder() throws Exception {
+
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setStrings(DFS_CBLOCK_ISCSI_ADVERTISED_IP, "1.2.3.4");
+
+ DynamicProvisioner provisioner =
+ new DynamicProvisioner(conf, null);
+
+ String pvc = new String(Files.readAllBytes(
+ Paths.get(getClass().getResource(
+ "/dynamicprovisioner/input1-pvc.json").toURI())));
+
+ String pv = new String(Files.readAllBytes(
+ Paths.get(getClass().getResource(
+ "/dynamicprovisioner/expected1-pv.json").toURI())));
+
+ JSON json = new io.kubernetes.client.JSON();
+
+ V1PersistentVolumeClaim claim =
+ json.getGson().fromJson(pvc, V1PersistentVolumeClaim.class);
+
+ String volumeName = provisioner.createVolumeName(claim);
+
+ V1PersistentVolume volume =
+ provisioner.persitenceVolumeBuilder(claim, volumeName);
+
+ //remove the data which should not been compared
+ V1PersistentVolume expectedVolume =
+ json.getGson().fromJson(pv, V1PersistentVolume.class);
+
+
+ Assert.assertEquals(expectedVolume, volume);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
new file mode 100644
index 0000000..8cb57d6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/ContainerLookUpService.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.ozone.container.ContainerTestHelper;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * NOTE : This class is only for testing purpose.
+ *
+ * Mock an underlying container storage layer, expose to CBlock to perform
+ * IO. While in this mock implementation, a container is nothing more than
+ * a in memory hashmap.
+ *
+ * This is to allow volume creation call and perform standalone tests.
+ */
+public final class ContainerLookUpService {
+ private static ConcurrentHashMap<String, ContainerDescriptor>
+ containers = new ConcurrentHashMap<>();
+
+ /**
+ * Return an *existing* container with given Id.
+ *
+ * TODO : for testing purpose, return a new container if the given Id
+ * is not found
+ *
+ * found
+ * @param containerID
+ * @return the corresponding pipeline instance (create if not exist)
+ */
+ public static ContainerDescriptor lookUp(String containerID)
+ throws IOException {
+ if (!containers.containsKey(containerID)) {
+ Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
+ containerID);
+ ContainerDescriptor cd = new ContainerDescriptor(containerID);
+ cd.setPipeline(pipeline);
+ containers.put(containerID, cd);
+ }
+ return containers.get(containerID);
+ }
+
+ public static void addContainer(String containerID) throws IOException {
+ Pipeline pipeline = ContainerTestHelper.createSingleNodePipeline(
+ containerID);
+ ContainerDescriptor cd = new ContainerDescriptor(containerID);
+ cd.setPipeline(pipeline);
+ containers.put(containerID, cd);
+ }
+
+ private ContainerLookUpService() {
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
new file mode 100644
index 0000000..59c8e01
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/util/MockStorageClient.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos.ContainerData;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class is the one that directly talks to SCM server.
+ *
+ * NOTE : this is only a mock class, only to allow testing volume
+ * creation without actually creating containers. In real world, need to be
+ * replaced with actual container look up calls.
+ *
+ */
+public class MockStorageClient implements ScmClient {
+ private static AtomicInteger currentContainerId =
+ new AtomicInteger(0);
+
+ /**
+ * Ask SCM to get a exclusive container.
+ *
+ * @return A container descriptor object to locate this container
+ * @throws Exception
+ */
+ @Override
+ public Pipeline createContainer(String containerId, String owner)
+ throws IOException {
+ int contId = currentContainerId.getAndIncrement();
+ ContainerLookUpService.addContainer(Long.toString(contId));
+ return ContainerLookUpService.lookUp(Long.toString(contId))
+ .getPipeline();
+ }
+
+ /**
+ * As this is only a testing class, with all "container" maintained in
+ * memory, no need to really delete anything for now.
+ * @throws IOException
+ */
+ @Override
+ public void deleteContainer(Pipeline pipeline, boolean force)
+ throws IOException {
+
+ }
+
+ /**
+ * This is a mock class, so returns the container infos of start container
+ * and end container.
+ *
+ * @param startName start container name.
+ * @param prefixName prefix container name.
+ * @param count count.
+ * @return a list of pipeline.
+ * @throws IOException
+ */
+ @Override
+ public List<ContainerInfo> listContainer(String startName,
+ String prefixName, int count) throws IOException {
+ List<ContainerInfo> containerList = new ArrayList<>();
+ ContainerDescriptor containerDescriptor =
+ ContainerLookUpService.lookUp(startName);
+ ContainerInfo container = new ContainerInfo.Builder()
+ .setContainerName(containerDescriptor.getContainerID())
+ .setPipeline(containerDescriptor.getPipeline())
+ .setState(HdslProtos.LifeCycleState.ALLOCATED)
+ .build();
+ containerList.add(container);
+ return containerList;
+ }
+
+ /**
+ * Create a instance of ContainerData by a given container id,
+ * since this is a testing class, there is no need set up the hold
+ * env to get the meta data of the container.
+ * @param pipeline
+ * @return
+ * @throws IOException
+ */
+ @Override
+ public ContainerData readContainer(Pipeline pipeline) throws IOException {
+ return ContainerData.newBuilder()
+ .setName(pipeline.getContainerName())
+ .build();
+ }
+
+ /**
+ * Return reference to an *existing* container with given ID.
+ *
+ * @param containerId
+ * @return
+ * @throws IOException
+ */
+ public Pipeline getContainer(String containerId)
+ throws IOException {
+ return ContainerLookUpService.lookUp(containerId).getPipeline();
+ }
+
+ @Override
+ public void closeContainer(Pipeline container) throws IOException {
+ // Do nothing, because the mock container does not have the notion of
+ // "open" and "close".
+ }
+
+ @Override
+ public long getContainerSize(Pipeline pipeline) throws IOException {
+ // just return a constant value for now
+ return 5L * OzoneConsts.GB; // 5GB
+ }
+
+ @Override
+ public Pipeline createContainer(HdslProtos.ReplicationType type,
+ HdslProtos.ReplicationFactor replicationFactor, String containerId,
+ String owner) throws IOException {
+ int contId = currentContainerId.getAndIncrement();
+ ContainerLookUpService.addContainer(Long.toString(contId));
+ return ContainerLookUpService.lookUp(Long.toString(contId))
+ .getPipeline();
+ }
+
+ /**
+ * Returns a set of Nodes that meet a query criteria.
+ *
+ * @param nodeStatuses - A set of criteria that we want the node to have.
+ * @param queryScope - Query scope - Cluster or pool.
+ * @param poolName - if it is pool, a pool name is required.
+ * @return A set of nodes that meet the requested criteria.
+ * @throws IOException
+ */
+ @Override
+ public HdslProtos.NodePool queryNode(EnumSet<HdslProtos.NodeState>
+ nodeStatuses, HdslProtos.QueryScope queryScope, String poolName)
+ throws IOException {
+ return null;
+ }
+
+ /**
+ * Creates a specified replication pipeline.
+ *
+ * @param type - Type
+ * @param factor - Replication factor
+ * @param nodePool - Set of machines.
+ * @throws IOException
+ */
+ @Override
+ public Pipeline createReplicationPipeline(HdslProtos.ReplicationType type,
+ HdslProtos.ReplicationFactor factor, HdslProtos.NodePool nodePool)
+ throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
new file mode 100644
index 0000000..2f8a193
--- /dev/null
+++ b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/expected1-pv.json
@@ -0,0 +1,37 @@
+{
+ "apiVersion": "v1",
+ "kind": "PersistentVolume",
+ "metadata": {
+ "annotations": {
+ "volume.beta.kubernetes.io/storage-class": "cblock",
+ "pv.kubernetes.io/provisioned-by": "hadoop.apache.org/cblock"
+ },
+ "name": "volume1-b65d053d-f92e-11e7-be3b-84b261c34638",
+ "namespace": "ns"
+ },
+ "spec": {
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "capacity": {
+ "storage": "1Gi"
+ },
+ "claimRef": {
+ "apiVersion": "v1",
+ "kind": "PersistentVolumeClaim",
+ "name": "volume1",
+ "namespace": "ns",
+ "uid": "b65d053d-f92e-11e7-be3b-84b261c34638"
+ },
+ "iscsi": {
+ "fsType": "ext4",
+ "iqn": "iqn.2001-04.org.apache.hadoop:volume1-b65d053d-f92e-11e7-be3b-84b261c34638",
+ "lun": 0,
+ "portals": [
+ "1.2.3.4:3260"
+ ],
+ "targetPortal": "1.2.3.4:3260"
+ },
+ "persistentVolumeReclaimPolicy": "Delete"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
new file mode 100644
index 0000000..e40ba71
--- /dev/null
+++ b/hadoop-cblock/server/src/test/resources/dynamicprovisioner/input1-pvc.json
@@ -0,0 +1,38 @@
+{
+ "apiVersion": "v1",
+ "kind": "PersistentVolumeClaim",
+ "metadata": {
+ "annotations": {
+ "pv.kubernetes.io/bind-completed": "yes",
+ "pv.kubernetes.io/bound-by-controller": "yes",
+ "volume.beta.kubernetes.io/storage-provisioner": "hadoop.apache.org/cblock"
+ },
+ "creationTimestamp": "2018-01-14T13:27:48Z",
+ "name": "volume1",
+ "namespace": "ns",
+ "resourceVersion": "5532691",
+ "selfLink": "/api/v1/namespaces/demo1/persistentvolumeclaims/persistent",
+ "uid": "b65d053d-f92e-11e7-be3b-84b261c34638"
+ },
+ "spec": {
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "resources": {
+ "requests": {
+ "storage": "1Gi"
+ }
+ },
+ "storageClassName": "cblock",
+ "volumeName": "persistent-b65d053d-f92e-11e7-be3b-84b261c34638"
+ },
+ "status": {
+ "accessModes": [
+ "ReadWriteOnce"
+ ],
+ "capacity": {
+ "storage": "1Gi"
+ },
+ "phase": "Bound"
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/pom.xml b/hadoop-cblock/tools/pom.xml
new file mode 100644
index 0000000..150f2e5
--- /dev/null
+++ b/hadoop-cblock/tools/pom.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cblock</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-cblock-tools</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <description>Apache Hadoop CBlock Tools</description>
+ <name>Apache Hadoop CBlock Tools</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hadoop.component>cblock</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cblock-server</artifactId>
+ </dependency>
+ </dependencies>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
new file mode 100644
index 0000000..224c908
--- /dev/null
+++ b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.cli;
+
+import org.apache.commons.cli.BasicParser;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.client.CBlockVolumeClient;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * The command line tool class.
+ */
+public class CBlockCli extends Configured implements Tool {
+
+ private static final String CREATE_VOLUME = "createVolume";
+
+ private static final String DELETE_VOLUME = "deleteVolume";
+
+ private static final String INFO_VOLUME = "infoVolume";
+
+ private static final String LIST_VOLUME = "listVolume";
+
+ private static final String SERVER_ADDR = "serverAddr";
+
+ private static final String HELP = "help";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CBlockCli.class);
+ private OzoneConfiguration conf;
+
+ private PrintStream printStream;
+
+ private Options options;
+
+ private BasicParser parser;
+
+ private CBlockVolumeClient localProxy;
+
+ public CBlockCli(OzoneConfiguration conf, PrintStream printStream)
+ throws IOException {
+ this.printStream = printStream;
+ this.conf = conf;
+ this.options = getOptions();
+ this.parser = new BasicParser();
+ }
+
+ public CBlockCli(OzoneConfiguration conf) throws IOException{
+ this(conf, System.out);
+ }
+
+ private CommandLine parseArgs(String[] argv)
+ throws ParseException {
+ return parser.parse(options, argv);
+ }
+
+ private static Options getOptions() {
+ Options options = new Options();
+ Option serverAddress = OptionBuilder
+ .withArgName("serverAddress>:<serverPort")
+ .withLongOpt(SERVER_ADDR)
+ .withValueSeparator(':')
+ .hasArgs(2)
+ .withDescription("specify server address:port")
+ .create("s");
+ options.addOption(serverAddress);
+
+ // taking 4 args: userName, volumeName, volumeSize, blockSize
+ Option createVolume = OptionBuilder
+ .withArgName("user> <volume> <volumeSize in [GB/TB]> <blockSize")
+ .withLongOpt(CREATE_VOLUME)
+ .withValueSeparator(' ')
+ .hasArgs(4)
+ .withDescription("create a fresh new volume")
+ .create("c");
+ options.addOption(createVolume);
+
+ // taking 2 args: userName, volumeName
+ Option deleteVolume = OptionBuilder
+ .withArgName("user> <volume")
+ .withLongOpt(DELETE_VOLUME)
+ .hasArgs(2)
+ .withDescription("delete a volume")
+ .create("d");
+ options.addOption(deleteVolume);
+
+ // taking 2 args: userName, volumeName
+ Option infoVolume = OptionBuilder
+ .withArgName("user> <volume")
+ .withLongOpt(INFO_VOLUME)
+ .hasArgs(2)
+ .withDescription("info a volume")
+ .create("i");
+ options.addOption(infoVolume);
+
+ // taking 1 arg: userName
+ Option listVolume = OptionBuilder
+ .withArgName("user")
+ .withLongOpt(LIST_VOLUME)
+ .hasOptionalArgs(1)
+ .withDescription("list all volumes")
+ .create("l");
+ options.addOption(listVolume);
+
+ Option help = OptionBuilder
+ .withLongOpt(HELP)
+ .withDescription("help")
+ .create("h");
+ options.addOption(help);
+
+ return options;
+ }
+
+ @Override
+ public int run(String[] args) throws ParseException, IOException {
+ CommandLine commandLine = parseArgs(args);
+ if (commandLine.hasOption("s")) {
+ String[] serverAddrArgs = commandLine.getOptionValues("s");
+ LOG.info("server address" + Arrays.toString(serverAddrArgs));
+ String serverHost = serverAddrArgs[0];
+ int serverPort = Integer.parseInt(serverAddrArgs[1]);
+ InetSocketAddress serverAddress =
+ new InetSocketAddress(serverHost, serverPort);
+ this.localProxy = new CBlockVolumeClient(conf, serverAddress);
+ } else {
+ this.localProxy = new CBlockVolumeClient(conf);
+ }
+
+ if (commandLine.hasOption("h")) {
+ LOG.info("help");
+ help();
+ }
+
+ if (commandLine.hasOption("c")) {
+ String[] createArgs = commandLine.getOptionValues("c");
+ LOG.info("create volume:" + Arrays.toString(createArgs));
+ createVolume(createArgs);
+ }
+
+ if (commandLine.hasOption("d")) {
+ String[] deleteArgs = commandLine.getOptionValues("d");
+ LOG.info("delete args:" + Arrays.toString(deleteArgs));
+ deleteVolume(deleteArgs);
+ }
+
+ if (commandLine.hasOption("l")) {
+ String[] listArg = commandLine.getOptionValues("l");
+ LOG.info("list args:" + Arrays.toString(listArg));
+ listVolume(listArg);
+ }
+
+ if (commandLine.hasOption("i")) {
+ String[] infoArgs = commandLine.getOptionValues("i");
+ LOG.info("info args:" + Arrays.toString(infoArgs));
+ infoVolume(infoArgs);
+ }
+ return 0;
+ }
+
+ public static void main(String[] argv) throws Exception {
+ CblockUtils.activateConfigs();
+ OzoneConfiguration cblockConf = new OzoneConfiguration();
+ RPC.setProtocolEngine(cblockConf, CBlockServiceProtocolPB.class,
+ ProtobufRpcEngine.class);
+ int res = 0;
+ Tool shell = new CBlockCli(cblockConf, System.out);
+ try {
+ ToolRunner.run(shell, argv);
+ } catch (Exception ex) {
+ LOG.error(ex.toString());
+ res = 1;
+ }
+ System.exit(res);
+ }
+
+
+
+ private void createVolume(String[] createArgs) throws IOException {
+ String userName = createArgs[0];
+ String volumeName = createArgs[1];
+ long volumeSize = CblockUtils.parseSize(createArgs[2]);
+ int blockSize = Integer.parseInt(createArgs[3])*1024;
+ localProxy.createVolume(userName, volumeName, volumeSize, blockSize);
+ }
+
+ private void deleteVolume(String[] deleteArgs) throws IOException {
+ String userName = deleteArgs[0];
+ String volumeName = deleteArgs[1];
+ boolean force = false;
+ if (deleteArgs.length > 2) {
+ force = Boolean.parseBoolean(deleteArgs[2]);
+ }
+ localProxy.deleteVolume(userName, volumeName, force);
+ }
+
+ private void infoVolume(String[] infoArgs) throws IOException {
+ String userName = infoArgs[0];
+ String volumeName = infoArgs[1];
+ VolumeInfo volumeInfo = localProxy.infoVolume(userName, volumeName);
+ printStream.println(volumeInfo.toString());
+ }
+
+ private void listVolume(String[] listArgs) throws IOException {
+ StringBuilder stringBuilder = new StringBuilder();
+ List<VolumeInfo> volumeResponse;
+ if (listArgs == null) {
+ volumeResponse = localProxy.listVolume(null);
+ } else {
+ volumeResponse = localProxy.listVolume(listArgs[0]);
+ }
+ for (int i = 0; i<volumeResponse.size(); i++) {
+ stringBuilder.append(
+ String.format("%s:%s\t%d\t%d", volumeResponse.get(i).getUserName(),
+ volumeResponse.get(i).getVolumeName(),
+ volumeResponse.get(i).getVolumeSize(),
+ volumeResponse.get(i).getBlockSize()));
+ if (i < volumeResponse.size() - 1) {
+ stringBuilder.append("\n");
+ }
+ }
+ printStream.println(stringBuilder);
+ }
+
+ private void help() {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(100, "cblock", "", options, "");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
new file mode 100644
index 0000000..b8b1889
--- /dev/null
+++ b/hadoop-cblock/tools/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.cli;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java b/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
new file mode 100644
index 0000000..f8b05ed
--- /dev/null
+++ b/hadoop-cblock/tools/src/test/org/apache/hadoop/cblock/TestCBlockCLI.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.cblock.cli.CBlockCli;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.util.MockStorageClient;
+import org.apache.hadoop.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * A testing class for cblock command line tool.
+ */
+public class TestCBlockCLI {
+ private static final long GB = 1 * 1024 * 1024 * 1024L;
+ private static final int KB = 1024;
+ private static CBlockCli cmd;
+ private static OzoneConfiguration conf;
+ private static CBlockManager cBlockManager;
+ private static ByteArrayOutputStream outContent;
+ private static PrintStream testPrintOut;
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ outContent = new ByteArrayOutputStream();
+ ScmClient storageClient = new MockStorageClient();
+ conf = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestCBlockCLI.class.getSimpleName());
+ File filePath = new File(path);
+ if (!filePath.exists() && !filePath.mkdirs()) {
+ throw new IOException("Unable to create test DB dir");
+ }
+ conf.set(DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, "127.0.0.1:0");
+ conf.set(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY, path.concat(
+ "/testCblockCli.dat"));
+ cBlockManager = new CBlockManager(conf, storageClient);
+ cBlockManager.start();
+ testPrintOut = new PrintStream(outContent);
+ cmd = new CBlockCli(conf, testPrintOut);
+ }
+
+ @AfterClass
+ public static void clean() {
+ if (cBlockManager != null) {
+ cBlockManager.stop();
+ cBlockManager.join();
+ cBlockManager.clean();
+ }
+ }
+
+ @After
+ public void reset() {
+ outContent.reset();
+ }
+
+ /**
+ * Test the help command.
+ * @throws Exception
+ */
+ @Test
+ public void testCliHelp() throws Exception {
+ PrintStream initialStdOut = System.out;
+ System.setOut(testPrintOut);
+ String[] args = {"-h"};
+ cmd.run(args);
+ String helpPrints =
+ "usage: cblock\n" +
+ " -c,--createVolume <user> <volume> <volumeSize in [GB/TB]> " +
+ "<blockSize> create a fresh new volume\n" +
+ " -d,--deleteVolume <user> <volume> " +
+ " delete a volume\n" +
+ " -h,--help " +
+ " help\n" +
+ " -i,--infoVolume <user> <volume> " +
+ " info a volume\n" +
+ " -l,--listVolume <user> " +
+ " list all volumes\n" +
+ " -s,--serverAddr <serverAddress>:<serverPort> " +
+ " specify server address:port\n";
+ assertEquals(helpPrints, outContent.toString());
+ outContent.reset();
+ System.setOut(initialStdOut);
+ }
+
+ /**
+ * Test volume listing command.
+ * @throws Exception
+ */
+ @Test
+ public void testCliList() throws Exception {
+ String userName0 = "userTestCliList0";
+ String userName1 = "userTestCliList1";
+ String userTestNotExist = "userTestNotExist";
+ String volumeName0 = "volumeTest0";
+ String volumeName1 = "volumeTest1";
+ String volumeSize0 = "30GB";
+ String volumeSize1 = "40GB";
+ String blockSize = Integer.toString(4);
+ String[] argsCreate0 =
+ {"-c", userName0, volumeName0, volumeSize0, blockSize};
+ cmd.run(argsCreate0);
+ String[] argsCreate1 =
+ {"-c", userName0, volumeName1, volumeSize1, blockSize};
+ cmd.run(argsCreate1);
+ String[] argsCreate2 =
+ {"-c", userName1, volumeName0, volumeSize0, blockSize};
+ cmd.run(argsCreate2);
+ String[] argsList0 = {"-l"};
+ cmd.run(argsList0);
+ String[] outExpected1 = {
+ "userTestCliList1:volumeTest0\t32212254720\t4096\n",
+ "userTestCliList0:volumeTest0\t32212254720\t4096\n",
+ "userTestCliList0:volumeTest1\t42949672960\t4096\n"};
+ int length = 0;
+ for (String str : outExpected1) {
+ assertTrue(outContent.toString().contains(str));
+ length += str.length();
+ }
+ assertEquals(length, outContent.toString().length());
+ outContent.reset();
+
+ String[] argsList1 = {"-l", userName1};
+ cmd.run(argsList1);
+ String outExpected2 = "userTestCliList1:volumeTest0\t32212254720\t4096\n";
+ assertEquals(outExpected2, outContent.toString());
+ outContent.reset();
+
+ String[] argsList2 = {"-l", userTestNotExist};
+ cmd.run(argsList2);
+ String outExpected3 = "\n";
+ assertEquals(outExpected3, outContent.toString());
+ }
+
+ /**
+ * Test create volume command.
+ * @throws Exception
+ */
+ @Test
+ public void testCliCreate() throws Exception {
+ String userName = "userTestCliCreate";
+ String volumeName = "volumeTest";
+ String volumeSize = "30GB";
+ String blockSize = "4";
+ String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
+ cmd.run(argsCreate);
+ List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
+ assertEquals(1, allVolumes.size());
+ VolumeDescriptor volume = allVolumes.get(0);
+ assertEquals(userName, volume.getUserName());
+ assertEquals(volumeName, volume.getVolumeName());
+ long volumeSizeB = volume.getVolumeSize();
+ assertEquals(30, (int)(volumeSizeB/ GB));
+ assertEquals(4, volume.getBlockSize()/ KB);
+ }
+
+ /**
+ * Test delete volume command.
+ * @throws Exception
+ */
+ @Test
+ public void testCliDelete() throws Exception {
+ String userName = "userTestCliDelete";
+ String volumeName = "volumeTest";
+ String volumeSize = "30GB";
+ String blockSize = "4";
+ String[] argsCreate = {"-c", userName, volumeName, volumeSize, blockSize};
+ cmd.run(argsCreate);
+ List<VolumeDescriptor> allVolumes = cBlockManager.getAllVolumes(userName);
+ assertEquals(1, allVolumes.size());
+ VolumeDescriptor volume = allVolumes.get(0);
+ assertEquals(userName, volume.getUserName());
+ assertEquals(volumeName, volume.getVolumeName());
+ long volumeSizeB = volume.getVolumeSize();
+ assertEquals(30, (int)(volumeSizeB/ GB));
+ assertEquals(4, volume.getBlockSize()/ KB);
+
+ String[] argsDelete = {"-d", userName, volumeName};
+ cmd.run(argsDelete);
+ allVolumes = cBlockManager.getAllVolumes(userName);
+ assertEquals(0, allVolumes.size());
+ }
+
+ /**
+ * Test info volume command.
+ * @throws Exception
+ */
+ @Test
+ public void testCliInfoVolume() throws Exception {
+ String userName0 = "userTestCliInfo";
+ String volumeName0 = "volumeTest0";
+ String volumeSize = "8000GB";
+ String blockSize = "4";
+ String[] argsCreate0 = {
+ "-c", userName0, volumeName0, volumeSize, blockSize};
+ cmd.run(argsCreate0);
+ String[] argsInfo = {"-i", userName0, volumeName0};
+ cmd.run(argsInfo);
+ // TODO : the usage field is not implemented yet, always 0 now.
+ String outExpected = " userName:userTestCliInfo " +
+ "volumeName:volumeTest0 " +
+ "volumeSize:8589934592000 " +
+ "blockSize:4096 (sizeInBlocks:2097152000) usageInBlocks:0\n";
+ assertEquals(outExpected, outContent.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 9ea4587..c3e7113 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -596,6 +596,13 @@ function hadoop_bootstrap
YARN_LIB_JARS_DIR=${YARN_LIB_JARS_DIR:-"share/hadoop/yarn/lib"}
MAPRED_DIR=${MAPRED_DIR:-"share/hadoop/mapreduce"}
MAPRED_LIB_JARS_DIR=${MAPRED_LIB_JARS_DIR:-"share/hadoop/mapreduce/lib"}
+ HDSL_DIR=${HDSL_DIR:-"share/hadoop/hdsl"}
+ HDSL_LIB_JARS_DIR=${HDSL_LIB_JARS_DIR:-"share/hadoop/hdsl/lib"}
+ OZONE_DIR=${OZONE_DIR:-"share/hadoop/ozone"}
+ OZONE_LIB_JARS_DIR=${OZONE_LIB_JARS_DIR:-"share/hadoop/ozone/lib"}
+ CBLOCK_DIR=${CBLOCK_DIR:-"share/hadoop/cblock"}
+ CBLOCK_LIB_JARS_DIR=${CBLOCK_LIB_JARS_DIR:-"share/hadoop/cblock/lib"}
+
HADOOP_TOOLS_HOME=${HADOOP_TOOLS_HOME:-${HADOOP_HOME}}
HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
index b19e54d..2128de7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/ConfServlet.java
@@ -17,15 +17,9 @@
*/
package org.apache.hadoop.conf;
-import com.google.gson.Gson;
import java.io.IOException;
import java.io.Writer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.servlet.http.HttpServlet;
@@ -38,8 +32,6 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.http.HttpServer2;
import com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A servlet to print out the running configuration data.
@@ -51,12 +43,6 @@ public class ConfServlet extends HttpServlet {
protected static final String FORMAT_JSON = "json";
protected static final String FORMAT_XML = "xml";
- private static final String COMMAND = "cmd";
- private static final Logger LOG = LoggerFactory.getLogger(ConfServlet.class);
- private transient static final Configuration OZONE_CONFIG = new
- OzoneConfiguration();
- private transient Map<String, OzoneConfiguration.Property> propertyMap = null;
-
/**
* Return the Configuration of the daemon hosting this servlet.
@@ -79,27 +65,21 @@ public class ConfServlet extends HttpServlet {
final ServletContext servletContext = getServletContext();
if (!HttpServer2.isStaticUserAndNoneAuthType(servletContext, request) &&
!HttpServer2.isInstrumentationAccessAllowed(servletContext,
- request, response)) {
+ request, response)) {
return;
}
String format = parseAcceptHeader(request);
- String cmd = request.getParameter(COMMAND);
- Writer out = response.getWriter();
+ if (FORMAT_XML.equals(format)) {
+ response.setContentType("text/xml; charset=utf-8");
+ } else if (FORMAT_JSON.equals(format)) {
+ response.setContentType("application/json; charset=utf-8");
+ }
+ String name = request.getParameter("name");
+ Writer out = response.getWriter();
try {
- if (cmd == null) {
- if (FORMAT_XML.equals(format)) {
- response.setContentType("text/xml; charset=utf-8");
- } else if (FORMAT_JSON.equals(format)) {
- response.setContentType("application/json; charset=utf-8");
- }
-
- String name = request.getParameter("name");
- writeResponse(getConfFromContext(), out, format, name);
- } else {
- processConfigTagRequest(request, out);
- }
+ writeResponse(getConfFromContext(), out, format, name);
} catch (BadFormatException bfe) {
response.sendError(HttpServletResponse.SC_BAD_REQUEST, bfe.getMessage());
} catch (IllegalArgumentException iae) {
@@ -108,67 +88,6 @@ public class ConfServlet extends HttpServlet {
out.close();
}
- private void processConfigTagRequest(HttpServletRequest request,
- Writer out) throws IOException {
- String cmd = request.getParameter(COMMAND);
- Gson gson = new Gson();
- Configuration config = getOzoneConfig();
-
- config.get("ozone.enabled");
- switch (cmd) {
- case "getPropertyByTag":
- String tags = request.getParameter("tags");
- String tagGroup = request.getParameter("group");
- LOG.debug("Getting all properties for tags:" + tags + " group:" +
- tagGroup);
- List<String> tagList = new ArrayList<>();
- for (String tag : tags.split(",")) {
- if (config.isPropertyTag(tag)) {
- tagList.add(tag);
- }
- }
-
- Properties properties = config.getAllPropertiesByTags(tagList);
- if (propertyMap == null) {
- loadDescriptions();
- }
-
- List<OzoneConfiguration.Property> filteredProperties = new ArrayList<>();
-
- properties.stringPropertyNames().stream().forEach(key -> {
- if (config.get(key) != null) {
- propertyMap.get(key).setValue(config.get(key));
- filteredProperties.add(propertyMap.get(key));
- }
- });
- out.write(gson.toJsonTree(filteredProperties).toString());
- break;
- default:
- throw new IllegalArgumentException(cmd + " is not a valid command.");
- }
-
- }
-
- private void loadDescriptions() {
- OzoneConfiguration config = (OzoneConfiguration) getOzoneConfig();
- List<OzoneConfiguration.Property> propList = null;
- propertyMap = new HashMap<>();
- try {
- propList = config.readPropertyFromXml(config.getResource("ozone-site"
- + ".xml"));
- propList.stream().map(p -> propertyMap.put(p.getName(), p));
- propList = config.readPropertyFromXml(config.getResource("ozone-default"
- + ".xml"));
- propList.stream().forEach(p -> {
- if (!propertyMap.containsKey(p.getName().trim())) {
- propertyMap.put(p.getName().trim(), p);
- }
- });
- } catch (Exception e) {
- LOG.error("Error while reading description from xml files", e);
- }
- }
-
@VisibleForTesting
static String parseAcceptHeader(HttpServletRequest request) {
String format = request.getHeader(HttpHeaders.ACCEPT);
@@ -204,8 +123,4 @@ public class ConfServlet extends HttpServlet {
}
}
- private static Configuration getOzoneConfig() {
- return OZONE_CONFIG;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
deleted file mode 100644
index ed118bf..0000000
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/OzoneConfiguration.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.conf;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.List;
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.JAXBException;
-import javax.xml.bind.Unmarshaller;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Configuration for ozone.
- */
-@InterfaceAudience.Private
-public class OzoneConfiguration extends Configuration {
- static {
- // adds the default resources
- Configuration.addDefaultResource("hdfs-default.xml");
- Configuration.addDefaultResource("hdfs-site.xml");
- Configuration.addDefaultResource("ozone-default.xml");
- Configuration.addDefaultResource("ozone-site.xml");
- }
-
- public OzoneConfiguration() {
- }
-
- public OzoneConfiguration(Configuration conf) {
- super(conf);
- }
-
- public List<Property> readPropertyFromXml(URL url) throws JAXBException {
- JAXBContext context = JAXBContext.newInstance(XMLConfiguration.class);
- Unmarshaller um = context.createUnmarshaller();
-
- XMLConfiguration config = (XMLConfiguration) um.unmarshal(url);
- return config.getProperties();
- }
-
- /**
- * Class to marshall/un-marshall configuration from xml files.
- */
- @XmlAccessorType(XmlAccessType.FIELD)
- @XmlRootElement(name = "configuration")
- public static class XMLConfiguration {
-
- @XmlElement(name = "property", type = Property.class)
- private List<Property> properties = new ArrayList<>();
-
- public XMLConfiguration() {
- }
-
- public XMLConfiguration(List<Property> properties) {
- this.properties = properties;
- }
-
- public List<Property> getProperties() {
- return properties;
- }
-
- public void setProperties(List<Property> properties) {
- this.properties = properties;
- }
- }
-
- /**
- * Class to marshall/un-marshall configuration properties from xml files.
- */
- @XmlAccessorType(XmlAccessType.FIELD)
- @XmlRootElement(name = "property")
- public static class Property implements Comparable<Property> {
-
- private String name;
- private String value;
- private String tag;
- private String description;
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getValue() {
- return value;
- }
-
- public void setValue(String value) {
- this.value = value;
- }
-
- public String getTag() {
- return tag;
- }
-
- public void setTag(String tag) {
- this.tag = tag;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- @Override
- public int compareTo(Property o) {
- if (this == o) {
- return 0;
- }
- return this.getName().compareTo(o.getName());
- }
-
- @Override
- public String toString() {
- return this.getName() + " " + this.getValue() + this.getTag();
- }
-
- @Override
- public int hashCode(){
- return this.getName().hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- return (obj instanceof Property) && (((Property) obj).getName())
- .equals(this.getName());
- }
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
deleted file mode 100644
index 00d4857..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/Header.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.headers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * OZONE specific HTTP headers.
- */
-@InterfaceAudience.Private
-public final class Header {
- public static final String OZONE_QUOTA_BYTES = "BYTES";
- public static final String OZONE_QUOTA_MB = "MB";
- public static final String OZONE_QUOTA_GB = "GB";
- public static final String OZONE_QUOTA_TB = "TB";
- public static final String OZONE_QUOTA_REMOVE = "remove";
- public static final String OZONE_QUOTA_UNDEFINED = "undefined";
- public static final String OZONE_EMPTY_STRING="";
- public static final String OZONE_DEFAULT_LIST_SIZE = "1000";
-
- public static final String OZONE_USER = "x-ozone-user";
- public static final String OZONE_SIMPLE_AUTHENTICATION_SCHEME = "OZONE";
- public static final String OZONE_VERSION_HEADER = "x-ozone-version";
- public static final String OZONE_V1_VERSION_HEADER ="v1";
-
- public static final String OZONE_LIST_QUERY_SERVICE = "service";
-
- public static final String OZONE_INFO_QUERY_VOLUME = "volume";
- public static final String OZONE_INFO_QUERY_BUCKET = "bucket";
- public static final String OZONE_INFO_QUERY_KEY = "key";
-
- public static final String OZONE_REQUEST_ID = "x-ozone-request-id";
- public static final String OZONE_SERVER_NAME = "x-ozone-server-name";
-
- public static final String OZONE_STORAGE_TYPE = "x-ozone-storage-type";
-
- public static final String OZONE_BUCKET_VERSIONING =
- "x-ozone-bucket-versioning";
-
- public static final String OZONE_ACLS = "x-ozone-acls";
- public static final String OZONE_ACL_ADD = "ADD";
- public static final String OZONE_ACL_REMOVE = "REMOVE";
-
- public static final String OZONE_INFO_QUERY_TAG ="info";
- public static final String OZONE_QUOTA_QUERY_TAG ="quota";
- public static final String CONTENT_MD5 = "Content-MD5";
- public static final String OZONE_LIST_QUERY_PREFIX="prefix";
- public static final String OZONE_LIST_QUERY_MAXKEYS="max-keys";
- public static final String OZONE_LIST_QUERY_PREVKEY="prev-key";
- public static final String OZONE_LIST_QUERY_ROOTSCAN="root-scan";
-
- private Header() {
- // Never constructed.
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
deleted file mode 100644
index 340709f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/headers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.client.rest.headers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
deleted file mode 100644
index ebcc104..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest;
-
-/**
- * This package contains Ozone rest client library classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
deleted file mode 100644
index 3c65df5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/BucketInfo.java
+++ /dev/null
@@ -1,230 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.rest.response;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * BucketInfo class is used used for parsing json response
- * when BucketInfo Call is made.
- */
-public class BucketInfo implements Comparable<BucketInfo> {
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(BucketInfo.class);
-
- private String volumeName;
- private String bucketName;
- private String createdOn;
- private List<OzoneAcl> acls;
- private OzoneConsts.Versioning versioning;
- private StorageType storageType;
-
- /**
- * Constructor for BucketInfo.
- *
- * @param volumeName
- * @param bucketName
- */
- public BucketInfo(String volumeName, String bucketName) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- }
-
-
- /**
- * Default constructor for BucketInfo.
- */
- public BucketInfo() {
- acls = new LinkedList<>();
- }
-
- /**
- * Parse a JSON string into BucketInfo Object.
- *
- * @param jsonString Json String
- * @return BucketInfo
- * @throws IOException
- */
- public static BucketInfo parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
- /**
- * Returns a List of ACLs set on the Bucket.
- *
- * @return List of Acl
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Sets ACls.
- *
- * @param acls Acl list
- */
- public void setAcls(List<OzoneAcl> acls) {
- this.acls = acls;
- }
-
- /**
- * Returns Storage Type info.
- *
- * @return Storage Type of the bucket
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Sets the Storage Type.
- *
- * @param storageType Storage Type
- */
- public void setStorageType(StorageType storageType) {
- this.storageType = storageType;
- }
-
- /**
- * Returns versioning.
- *
- * @return versioning Enum
- */
- public OzoneConsts.Versioning getVersioning() {
- return versioning;
- }
-
- /**
- * Sets Versioning.
- *
- * @param versioning
- */
- public void setVersioning(OzoneConsts.Versioning versioning) {
- this.versioning = versioning;
- }
-
-
- /**
- * Gets bucket Name.
- *
- * @return String
- */
- public String getBucketName() {
- return bucketName;
- }
-
- /**
- * Sets bucket Name.
- *
- * @param bucketName Name of the bucket
- */
- public void setBucketName(String bucketName) {
- this.bucketName = bucketName;
- }
-
- /**
- * Sets creation time of the bucket.
- *
- * @param creationTime Date String
- */
- public void setCreatedOn(String creationTime) {
- this.createdOn = creationTime;
- }
-
- /**
- * Returns creation time.
- *
- * @return creation time of bucket.
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * Returns Volume Name.
- *
- * @return String volume name
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Sets the Volume Name of bucket.
- *
- * @param volumeName volumeName
- */
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * Please note : BucketInfo compare functions are used only within the
- * context of a volume, hence volume name is purposefully ignored in
- * compareTo, equal and hashcode functions of this class.
- */
- @Override
- public int compareTo(BucketInfo o) {
- Preconditions.checkState(o.getVolumeName().equals(this.getVolumeName()));
- return this.bucketName.compareTo(o.getBucketName());
- }
-
- /**
- * Checks if two bucketInfo's are equal.
- * @param o Object BucketInfo
- * @return True or False
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof BucketInfo)) {
- return false;
- }
-
- BucketInfo that = (BucketInfo) o;
- Preconditions.checkState(that.getVolumeName().equals(this.getVolumeName()));
- return bucketName.equals(that.bucketName);
-
- }
-
- /**
- * Hash Code for this object.
- * @return int
- */
- @Override
- public int hashCode() {
- return bucketName.hashCode();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
deleted file mode 100644
index 5699fda..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/KeyInfo.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-
-import java.io.IOException;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-
-/**
- * KeyInfo class is used used for parsing json response
- * when KeyInfo Call is made.
- */
-public class KeyInfo implements Comparable<KeyInfo> {
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(KeyInfo.class);
-
- private long version;
- private String md5hash;
- private String createdOn;
- private String modifiedOn;
- private long size;
- private String keyName;
-
- /**
- * When this key was created.
- *
- * @return Date String
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * When this key was modified.
- *
- * @return Date String
- */
- public String getModifiedOn() {
- return modifiedOn;
- }
-
- /**
- * When this key was created.
- *
- * @param createdOn Date String
- */
- public void setCreatedOn(String createdOn) {
- this.createdOn = createdOn;
- }
-
- /**
- * When this key was modified.
- *
- * @param modifiedOn Date String
- */
- public void setModifiedOn(String modifiedOn) {
- this.modifiedOn = modifiedOn;
- }
-
- /**
- * Gets the Key name of this object.
- *
- * @return String
- */
- public String getKeyName() {
- return keyName;
- }
-
- /**
- * Sets the Key name of this object.
- *
- * @param keyName String
- */
- public void setKeyName(String keyName) {
- this.keyName = keyName;
- }
-
- /**
- * Returns the MD5 Hash for the data of this key.
- *
- * @return String MD5
- */
- public String getMd5hash() {
- return md5hash;
- }
-
- /**
- * Sets the MD5 value of this key.
- *
- * @param md5hash Md5 of this file
- */
- public void setMd5hash(String md5hash) {
- this.md5hash = md5hash;
- }
-
- /**
- * Number of bytes stored in the data part of this key.
- *
- * @return long size of the data file
- */
- public long getSize() {
- return size;
- }
-
- /**
- * Sets the size of the data part of this key.
- *
- * @param size Size in long
- */
- public void setSize(long size) {
- this.size = size;
- }
-
- /**
- * Version of this key.
- *
- * @return returns the version of this key.
- */
- public long getVersion() {
- return version;
- }
-
- /**
- * Sets the version of this key.
- *
- * @param version - Version String
- */
- public void setVersion(long version) {
- this.version = version;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object
- * is less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(KeyInfo o) {
- if (this.keyName.compareTo(o.getKeyName()) != 0) {
- return this.keyName.compareTo(o.getKeyName());
- }
-
- if (this.getVersion() == o.getVersion()) {
- return 0;
- }
- if (this.getVersion() < o.getVersion()) {
- return -1;
- }
- return 1;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- KeyInfo keyInfo = (KeyInfo) o;
-
- return new EqualsBuilder()
- .append(version, keyInfo.version)
- .append(keyName, keyInfo.keyName)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 37)
- .append(version)
- .append(keyName)
- .toHashCode();
- }
-
- /**
- * Parse a string to return KeyInfo Object.
- *
- * @param jsonString Json String
- * @return keyInfo
- * @throws IOException
- */
- public static KeyInfo parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
deleted file mode 100644
index 8db3ac2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeInfo.java
+++ /dev/null
@@ -1,215 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-
-import java.io.IOException;
-
-/**
- * VolumeInfo Class is used for parsing json response
- * when VolumeInfo Call is made.
- */
-@InterfaceAudience.Private
-public class VolumeInfo implements Comparable<VolumeInfo> {
-
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(VolumeInfo.class);
-
- private VolumeOwner owner;
- private OzoneQuota quota;
- private String volumeName;
- private String createdOn;
- private String createdBy;
-
-
- /**
- * Constructor for VolumeInfo.
- *
- * @param volumeName - Name of the Volume
- * @param createdOn _ Date String
- * @param createdBy - Person who created it
- */
- public VolumeInfo(String volumeName, String createdOn,
- String createdBy) {
- this.volumeName = volumeName;
- this.createdOn = createdOn;
- this.createdBy = createdBy;
- }
-
- /**
- * Constructor for VolumeInfo.
- */
- public VolumeInfo() {
- }
-
- /**
- * gets the volume name.
- *
- * @return Volume Name
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Sets the volume name.
- *
- * @param volumeName Volume Name
- */
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
-
- /**
- * Returns the name of the person who created this volume.
- *
- * @return Name of Admin who created this
- */
- public String getCreatedBy() {
- return createdBy;
- }
-
- /**
- * Sets the user name of the person who created this volume.
- *
- * @param createdBy UserName
- */
- public void setCreatedBy(String createdBy) {
- this.createdBy = createdBy;
- }
-
- /**
- * Gets the date on which this volume was created.
- *
- * @return Date String
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * Sets the date string.
- *
- * @param createdOn Date String
- */
- public void setCreatedOn(String createdOn) {
- this.createdOn = createdOn;
- }
-
- /**
- * Returns the owner info.
- *
- * @return OwnerInfo
- */
- public VolumeOwner getOwner() {
- return owner;
- }
-
- /**
- * Sets the owner.
- *
- * @param owner OwnerInfo
- */
- public void setOwner(VolumeOwner owner) {
- this.owner = owner;
- }
-
- /**
- * Returns the quota information on a volume.
- *
- * @return Quota
- */
- public OzoneQuota getQuota() {
- return quota;
- }
-
- /**
- * Sets the quota info.
- *
- * @param quota Quota Info
- */
- public void setQuota(OzoneQuota quota) {
- this.quota = quota;
- }
-
- /**
- * Comparable Interface.
- * @param o VolumeInfo Object.
- * @return Result of comparison
- */
- @Override
- public int compareTo(VolumeInfo o) {
- return this.volumeName.compareTo(o.getVolumeName());
- }
-
- /**
- * Returns VolumeInfo class from json string.
- *
- * @param data Json String
- *
- * @return VolumeInfo
- *
- * @throws IOException
- */
- public static VolumeInfo parse(String data) throws IOException {
- return READER.readValue(data);
- }
-
- /**
- * Indicates whether some other object is "equal to" this one.
- *
- * @param obj the reference object with which to compare.
- *
- * @return {@code true} if this object is the same as the obj
- * argument; {@code false} otherwise.
- */
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- VolumeInfo otherInfo = (VolumeInfo) obj;
- return otherInfo.getVolumeName().equals(this.getVolumeName());
- }
-
- /**
- * Returns a hash code value for the object. This method is
- * supported for the benefit of hash tables such as those provided by
- * HashMap.
- * @return a hash code value for this object.
- *
- * @see Object#equals(Object)
- * @see System#identityHashCode
- */
- @Override
- public int hashCode() {
- return getVolumeName().hashCode();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
deleted file mode 100644
index 132bc2e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/VolumeOwner.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Volume Owner represents the owner of a volume.
- *
- * This is a class instead of a string since we might need to extend this class
- * to support other forms of authentication.
- */
-@InterfaceAudience.Private
-public class VolumeOwner {
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private String name;
-
- /**
- * Constructor for VolumeOwner.
- *
- * @param name name of the User
- */
- public VolumeOwner(String name) {
- this.name = name;
- }
-
- /**
- * Constructs Volume Owner.
- */
- public VolumeOwner() {
- name = null;
- }
-
- /**
- * Returns the user name.
- *
- * @return Name
- */
- public String getName() {
- return name;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
deleted file mode 100644
index 432b029b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rest/response/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.hadoop.ozone.client.rest.response;
-
-/**
- * This package contains class for ozone rest client library.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
deleted file mode 100644
index 7b48f49..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++ /dev/null
@@ -1,567 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.BucketArgs;
-import org.apache.hadoop.ozone.client.OzoneBucket;
-import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneQuota;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.ReplicationFactor;
-import org.apache.hadoop.ozone.client.ReplicationType;
-import org.apache.hadoop.ozone.client.VolumeArgs;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocolPB
- .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.ksm.protocolPB
- .KeySpaceManagerProtocolPB;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServicePort;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-import java.util.stream.Collectors;
-
-/**
- * Ozone RPC Client Implementation, it connects to KSM, SCM and DataNode
- * to execute client calls. This uses RPC protocol for communication
- * with the servers.
- */
-public class RpcClient implements ClientProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RpcClient.class);
-
- private final OzoneConfiguration conf;
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final KeySpaceManagerProtocolClientSideTranslatorPB
- keySpaceManagerClient;
- private final XceiverClientManager xceiverClientManager;
- private final int chunkSize;
- private final UserGroupInformation ugi;
- private final OzoneAcl.OzoneACLRights userRights;
- private final OzoneAcl.OzoneACLRights groupRights;
-
- /**
- * Creates RpcClient instance with the given configuration.
- * @param conf
- * @throws IOException
- */
- public RpcClient(Configuration conf) throws IOException {
- Preconditions.checkNotNull(conf);
- this.conf = new OzoneConfiguration(conf);
- this.ugi = UserGroupInformation.getCurrentUser();
- this.userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
- KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
- this.groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
- KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
- long ksmVersion =
- RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
- InetSocketAddress ksmAddress = OzoneClientUtils
- .getKsmAddressForClients(conf);
- RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
- ProtobufRpcEngine.class);
- this.keySpaceManagerClient =
- new KeySpaceManagerProtocolClientSideTranslatorPB(
- RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
- ksmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- long scmVersion =
- RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
- InetSocketAddress scmAddress = getScmAddressForClient();
- RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- this.storageContainerLocationClient =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- this.xceiverClientManager = new XceiverClientManager(conf);
-
- int configuredChunkSize = conf.getInt(
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- if(configuredChunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
- LOG.warn("The chunk size ({}) is not allowed to be more than"
- + " the maximum size ({}),"
- + " resetting to the maximum size.",
- configuredChunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
- } else {
- chunkSize = configuredChunkSize;
- }
- }
-
- private InetSocketAddress getScmAddressForClient() throws IOException {
- List<ServiceInfo> services = keySpaceManagerClient.getServiceList();
- ServiceInfo scmInfo = services.stream().filter(
- a -> a.getNodeType().equals(OzoneProtos.NodeType.SCM))
- .collect(Collectors.toList()).get(0);
- return NetUtils.createSocketAddr(scmInfo.getHostname()+ ":" +
- scmInfo.getPort(ServicePort.Type.RPC));
- }
-
- @Override
- public void createVolume(String volumeName) throws IOException {
- createVolume(volumeName, VolumeArgs.newBuilder().build());
- }
-
- @Override
- public void createVolume(String volumeName, VolumeArgs volArgs)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(volArgs);
-
- String admin = volArgs.getAdmin() == null ?
- ugi.getUserName() : volArgs.getAdmin();
- String owner = volArgs.getOwner() == null ?
- ugi.getUserName() : volArgs.getOwner();
- long quota = volArgs.getQuota() == null ?
- OzoneConsts.MAX_QUOTA_IN_BYTES :
- OzoneQuota.parseQuota(volArgs.getQuota()).sizeInBytes();
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- owner, userRights));
- //Group ACLs of the User
- List<String> userGroups = Arrays.asList(UserGroupInformation
- .createRemoteUser(owner).getGroupNames());
- userGroups.stream().forEach((group) -> listOfAcls.add(
- new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
- //ACLs from VolumeArgs
- if(volArgs.getAcls() != null) {
- listOfAcls.addAll(volArgs.getAcls());
- }
-
- KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
- builder.setVolume(volumeName);
- builder.setAdminName(admin);
- builder.setOwnerName(owner);
- builder.setQuotaInBytes(quota);
-
- //Remove duplicates and add ACLs
- for (OzoneAcl ozoneAcl :
- listOfAcls.stream().distinct().collect(Collectors.toList())) {
- builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(ozoneAcl));
- }
-
- LOG.info("Creating Volume: {}, with {} as owner and quota set to {} bytes.",
- volumeName, owner, quota);
- keySpaceManagerClient.createVolume(builder.build());
- }
-
- @Override
- public void setVolumeOwner(String volumeName, String owner)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(owner);
- keySpaceManagerClient.setOwner(volumeName, owner);
- }
-
- @Override
- public void setVolumeQuota(String volumeName, OzoneQuota quota)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(quota);
- long quotaInBytes = quota.sizeInBytes();
- keySpaceManagerClient.setQuota(volumeName, quotaInBytes);
- }
-
- @Override
- public OzoneVolume getVolumeDetails(String volumeName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- KsmVolumeArgs volume = keySpaceManagerClient.getVolumeInfo(volumeName);
- return new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
- }
-
- @Override
- public boolean checkVolumeAccess(String volumeName, OzoneAcl acl)
- throws IOException {
- throw new UnsupportedOperationException("Not yet implemented.");
- }
-
- @Override
- public void deleteVolume(String volumeName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- keySpaceManagerClient.deleteVolume(volumeName);
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String volumePrefix, String prevVolume,
- int maxListResult)
- throws IOException {
- List<KsmVolumeArgs> volumes = keySpaceManagerClient.listAllVolumes(
- volumePrefix, prevVolume, maxListResult);
-
- return volumes.stream().map(volume -> new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
- .collect(Collectors.toList());
- }
-
- @Override
- public List<OzoneVolume> listVolumes(String user, String volumePrefix,
- String prevVolume, int maxListResult)
- throws IOException {
- List<KsmVolumeArgs> volumes = keySpaceManagerClient.listVolumeByUser(
- user, volumePrefix, prevVolume, maxListResult);
-
- return volumes.stream().map(volume -> new OzoneVolume(
- conf,
- this,
- volume.getVolume(),
- volume.getAdminName(),
- volume.getOwnerName(),
- volume.getQuotaInBytes(),
- volume.getCreationTime(),
- volume.getAclMap().ozoneAclGetProtobuf().stream().
- map(KSMPBHelper::convertOzoneAcl).collect(Collectors.toList())))
- .collect(Collectors.toList());
- }
-
- @Override
- public void createBucket(String volumeName, String bucketName)
- throws IOException {
- createBucket(volumeName, bucketName, BucketArgs.newBuilder().build());
- }
-
- @Override
- public void createBucket(
- String volumeName, String bucketName, BucketArgs bucketArgs)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(bucketArgs);
-
- Boolean isVersionEnabled = bucketArgs.getVersioning() == null ?
- Boolean.FALSE : bucketArgs.getVersioning();
- StorageType storageType = bucketArgs.getStorageType() == null ?
- StorageType.DEFAULT : bucketArgs.getStorageType();
- List<OzoneAcl> listOfAcls = new ArrayList<>();
- //User ACL
- listOfAcls.add(new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- ugi.getUserName(), userRights));
- //Group ACLs of the User
- List<String> userGroups = Arrays.asList(UserGroupInformation
- .createRemoteUser(ugi.getUserName()).getGroupNames());
- userGroups.stream().forEach((group) -> listOfAcls.add(
- new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights)));
- //ACLs from BucketArgs
- if(bucketArgs.getAcls() != null) {
- listOfAcls.addAll(bucketArgs.getAcls());
- }
-
- KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setIsVersionEnabled(isVersionEnabled)
- .setStorageType(storageType)
- .setAcls(listOfAcls.stream().distinct().collect(Collectors.toList()));
-
- LOG.info("Creating Bucket: {}/{}, with Versioning {} and " +
- "Storage Type set to {}", volumeName, bucketName, isVersionEnabled,
- storageType);
- keySpaceManagerClient.createBucket(builder.build());
- }
-
- @Override
- public void addBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> addAcls)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(addAcls);
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setAddAcls(addAcls);
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void removeBucketAcls(
- String volumeName, String bucketName, List<OzoneAcl> removeAcls)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(removeAcls);
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setRemoveAcls(removeAcls);
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void setBucketVersioning(
- String volumeName, String bucketName, Boolean versioning)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(versioning);
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setIsVersionEnabled(versioning);
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void setBucketStorageType(
- String volumeName, String bucketName, StorageType storageType)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(storageType);
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setStorageType(storageType);
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void deleteBucket(
- String volumeName, String bucketName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- keySpaceManagerClient.deleteBucket(volumeName, bucketName);
- }
-
- @Override
- public void checkBucketAccess(
- String volumeName, String bucketName) throws IOException {
-
- }
-
- @Override
- public OzoneBucket getBucketDetails(
- String volumeName, String bucketName) throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- KsmBucketInfo bucketArgs =
- keySpaceManagerClient.getBucketInfo(volumeName, bucketName);
- return new OzoneBucket(
- conf,
- this,
- bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- bucketArgs.getAcls(),
- bucketArgs.getStorageType(),
- bucketArgs.getIsVersionEnabled(),
- bucketArgs.getCreationTime());
- }
-
- @Override
- public List<OzoneBucket> listBuckets(String volumeName, String bucketPrefix,
- String prevBucket, int maxListResult)
- throws IOException {
- List<KsmBucketInfo> buckets = keySpaceManagerClient.listBuckets(
- volumeName, prevBucket, bucketPrefix, maxListResult);
-
- return buckets.stream().map(bucket -> new OzoneBucket(
- conf,
- this,
- bucket.getVolumeName(),
- bucket.getBucketName(),
- bucket.getAcls(),
- bucket.getStorageType(),
- bucket.getIsVersionEnabled(),
- bucket.getCreationTime()))
- .collect(Collectors.toList());
- }
-
- @Override
- public OzoneOutputStream createKey(
- String volumeName, String bucketName, String keyName, long size,
- ReplicationType type, ReplicationFactor factor)
- throws IOException {
- String requestId = UUID.randomUUID().toString();
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .setDataSize(size)
- .setType(OzoneProtos.ReplicationType.valueOf(type.toString()))
- .setFactor(OzoneProtos.ReplicationFactor.valueOf(factor.getValue()))
- .build();
-
- OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
- ChunkGroupOutputStream groupOutputStream =
- new ChunkGroupOutputStream.Builder()
- .setHandler(openKey)
- .setXceiverClientManager(xceiverClientManager)
- .setScmClient(storageContainerLocationClient)
- .setKsmClient(keySpaceManagerClient)
- .setChunkSize(chunkSize)
- .setRequestID(requestId)
- .setType(OzoneProtos.ReplicationType.valueOf(type.toString()))
- .setFactor(OzoneProtos.ReplicationFactor.valueOf(factor.getValue()))
- .build();
- groupOutputStream.addPreallocateBlocks(
- openKey.getKeyInfo().getLatestVersionLocations(),
- openKey.getOpenVersion());
- return new OzoneOutputStream(groupOutputStream);
- }
-
- @Override
- public OzoneInputStream getKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- String requestId = UUID.randomUUID().toString();
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
- LengthInputStream lengthInputStream =
- ChunkGroupInputStream.getFromKsmKeyInfo(
- keyInfo, xceiverClientManager, storageContainerLocationClient,
- requestId);
- return new OzoneInputStream(
- (ChunkGroupInputStream)lengthInputStream.getWrappedStream());
- }
-
- @Override
- public void deleteKey(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- keySpaceManagerClient.deleteKey(keyArgs);
- }
-
- @Override
- public List<OzoneKey> listKeys(String volumeName, String bucketName,
- String keyPrefix, String prevKey,
- int maxListResult)
- throws IOException {
- List<KsmKeyInfo> keys = keySpaceManagerClient.listKeys(
- volumeName, bucketName, prevKey, keyPrefix, maxListResult);
-
- return keys.stream().map(key -> new OzoneKey(
- key.getVolumeName(),
- key.getBucketName(),
- key.getKeyName(),
- key.getDataSize(),
- key.getCreationTime(),
- key.getModificationTime()))
- .collect(Collectors.toList());
- }
-
- @Override
- public OzoneKey getKeyDetails(
- String volumeName, String bucketName, String keyName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(keyName);
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .setKeyName(keyName)
- .build();
- KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
- return new OzoneKey(keyInfo.getVolumeName(),
- keyInfo.getBucketName(),
- keyInfo.getKeyName(),
- keyInfo.getDataSize(),
- keyInfo.getCreationTime(),
- keyInfo.getModificationTime());
- }
-
- @Override
- public void close() throws IOException {
- IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
- IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
- IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
deleted file mode 100644
index 0fcc3fc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/rpc/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client.rpc;
-
-/**
- * This package contains Ozone rpc client library classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
deleted file mode 100644
index 5571f0a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/BlockGroup.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.common;
-
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.KeyBlocks;
-
-import java.util.List;
-
-/**
- * A group of blocks relations relevant, e.g belong to a certain object key.
- */
-public final class BlockGroup {
-
- private String groupID;
- private List<String> blockIDs;
- private BlockGroup(String groupID, List<String> blockIDs) {
- this.groupID = groupID;
- this.blockIDs = blockIDs;
- }
-
- public List<String> getBlockIDList() {
- return blockIDs;
- }
-
- public String getGroupID() {
- return groupID;
- }
-
- public KeyBlocks getProto() {
- return KeyBlocks.newBuilder().setKey(groupID)
- .addAllBlocks(blockIDs).build();
- }
-
- /**
- * Parses a KeyBlocks proto to a group of blocks.
- * @param proto KeyBlocks proto.
- * @return a group of blocks.
- */
- public static BlockGroup getFromProto(KeyBlocks proto) {
- return BlockGroup.newBuilder().setKeyName(proto.getKey())
- .addAllBlockIDs(proto.getBlocksList()).build();
- }
-
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * BlockGroup instance builder.
- */
- public static class Builder {
-
- private String groupID;
- private List<String> blockIDs;
-
- public Builder setKeyName(String blockGroupID) {
- this.groupID = blockGroupID;
- return this;
- }
-
- public Builder addAllBlockIDs(List<String> keyBlocks) {
- this.blockIDs = keyBlocks;
- return this;
- }
-
- public BlockGroup build() {
- return new BlockGroup(groupID, blockIDs);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
deleted file mode 100644
index f56f832..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/DeleteBlockGroupResult.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmBlockResult.Result;
-import org.apache.hadoop.scm.container.common.helpers.DeleteBlockResult;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Result to delete a group of blocks.
- */
-public class DeleteBlockGroupResult {
- private String objectKey;
- private List<DeleteBlockResult> blockResultList;
- public DeleteBlockGroupResult(String objectKey,
- List<DeleteBlockResult> blockResultList) {
- this.objectKey = objectKey;
- this.blockResultList = blockResultList;
- }
-
- public String getObjectKey() {
- return objectKey;
- }
-
- public List<DeleteBlockResult> getBlockResultList() {
- return blockResultList;
- }
-
- public List<DeleteScmBlockResult> getBlockResultProtoList() {
- List<DeleteScmBlockResult> resultProtoList =
- new ArrayList<>(blockResultList.size());
- for (DeleteBlockResult result : blockResultList) {
- DeleteScmBlockResult proto = DeleteScmBlockResult.newBuilder()
- .setKey(result.getKey())
- .setResult(result.getResult()).build();
- resultProtoList.add(proto);
- }
- return resultProtoList;
- }
-
- public static List<DeleteBlockResult> convertBlockResultProto(
- List<DeleteScmBlockResult> results) {
- List<DeleteBlockResult> protoResults = new ArrayList<>(results.size());
- for (DeleteScmBlockResult result : results) {
- protoResults.add(new DeleteBlockResult(result.getKey(),
- result.getResult()));
- }
- return protoResults;
- }
-
- /**
- * Only if all blocks are successfully deleted, this group is considered
- * to be successfully executed.
- *
- * @return true if all blocks are successfully deleted, false otherwise.
- */
- public boolean isSuccess() {
- for (DeleteBlockResult result : blockResultList) {
- if (result.getResult() != Result.success) {
- return false;
- }
- }
- return true;
- }
-
- /**
- * @return A list of deletion failed block IDs.
- */
- public List<String> getFailedBlocks() {
- List<String> failedBlocks = blockResultList.stream()
- .filter(result -> result.getResult() != Result.success)
- .map(DeleteBlockResult::getKey).collect(Collectors.toList());
- return failedBlocks;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
deleted file mode 100644
index 1e9fd59..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-/**
- ozone common shared by SCM, KSM, etc.
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
deleted file mode 100644
index 9aeff24..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/InvalidStateTransitionException.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.common.statemachine;
-
-/**
- * Class wraps invalid state transition exception.
- */
-public class InvalidStateTransitionException extends Exception {
- private Enum<?> currentState;
- private Enum<?> event;
-
- public InvalidStateTransitionException(Enum<?> currentState, Enum<?> event) {
- super("Invalid event: " + event + " at " + currentState + " state.");
- this.currentState = currentState;
- this.event = event;
- }
-
- public Enum<?> getCurrentState() {
- return currentState;
- }
-
- public Enum<?> getEvent() {
- return event;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
deleted file mode 100644
index bf8cbd5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/StateMachine.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.common.statemachine;
-
-import com.google.common.base.Supplier;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.CacheLoader;
-import com.google.common.cache.LoadingCache;
-
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * Template class that wraps simple event driven state machine.
- * @param <STATE> states allowed
- * @param <EVENT> events allowed
- */
-public class StateMachine<STATE extends Enum<?>, EVENT extends Enum<?>> {
- private STATE initialState;
- private Set<STATE> finalStates;
-
- private final LoadingCache<EVENT, Map<STATE, STATE>> transitions =
- CacheBuilder.newBuilder().build(
- CacheLoader.from((Supplier<Map<STATE, STATE>>) () -> new HashMap()));
-
- public StateMachine(STATE initState, Set<STATE> finalStates) {
- this.initialState = initState;
- this.finalStates = finalStates;
- }
-
- public STATE getInitialState() {
- return initialState;
- }
-
- public Set<STATE> getFinalStates() {
- return finalStates;
- }
-
- public STATE getNextState(STATE from, EVENT e)
- throws InvalidStateTransitionException {
- STATE target = transitions.getUnchecked(e).get(from);
- if (target == null) {
- throw new InvalidStateTransitionException(from, e);
- }
- return target;
- }
-
- public void addTransition(STATE from, STATE to, EVENT e) {
- transitions.getUnchecked(e).put(from, to);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
deleted file mode 100644
index 045409e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/common/statemachine/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common.statemachine;
-/**
- state machine template class for ozone.
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
deleted file mode 100644
index 75cf613..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/KSMConfigKeys.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-/**
- * KSM Constants.
- */
-public final class KSMConfigKeys {
- /**
- * Never constructed.
- */
- private KSMConfigKeys() {
- }
-
-
- public static final String OZONE_KSM_HANDLER_COUNT_KEY =
- "ozone.ksm.handler.count.key";
- public static final int OZONE_KSM_HANDLER_COUNT_DEFAULT = 20;
-
- public static final String OZONE_KSM_ADDRESS_KEY =
- "ozone.ksm.address";
- public static final String OZONE_KSM_BIND_HOST_DEFAULT =
- "0.0.0.0";
- public static final int OZONE_KSM_PORT_DEFAULT = 9862;
-
- public static final String OZONE_KSM_HTTP_ENABLED_KEY =
- "ozone.ksm.http.enabled";
- public static final String OZONE_KSM_HTTP_BIND_HOST_KEY =
- "ozone.ksm.http-bind-host";
- public static final String OZONE_KSM_HTTPS_BIND_HOST_KEY =
- "ozone.ksm.https-bind-host";
- public static final String OZONE_KSM_HTTP_ADDRESS_KEY =
- "ozone.ksm.http-address";
- public static final String OZONE_KSM_HTTPS_ADDRESS_KEY =
- "ozone.ksm.https-address";
- public static final String OZONE_KSM_KEYTAB_FILE =
- "ozone.ksm.keytab.file";
- public static final String OZONE_KSM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_KSM_HTTP_BIND_PORT_DEFAULT = 9874;
- public static final int OZONE_KSM_HTTPS_BIND_PORT_DEFAULT = 9875;
-
- // LevelDB cache file uses an off-heap cache in LevelDB of 128 MB.
- public static final String OZONE_KSM_DB_CACHE_SIZE_MB =
- "ozone.ksm.db.cache.size.mb";
- public static final int OZONE_KSM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_KSM_USER_MAX_VOLUME =
- "ozone.ksm.user.max.volume";
- public static final int OZONE_KSM_USER_MAX_VOLUME_DEFAULT = 1024;
-
- // KSM Default user/group permissions
- public static final String OZONE_KSM_USER_RIGHTS =
- "ozone.ksm.user.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_USER_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KSM_GROUP_RIGHTS =
- "ozone.ksm.group.rights";
- public static final OzoneAcl.OzoneACLRights OZONE_KSM_GROUP_RIGHTS_DEFAULT =
- OzoneAcl.OzoneACLRights.READ_WRITE;
-
- public static final String OZONE_KEY_DELETING_LIMIT_PER_TASK =
- "ozone.key.deleting.limit.per.task";
- public static final int OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT = 1000;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
deleted file mode 100644
index 1211b50..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketArgs.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.BucketArgs;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-
-/**
- * A class that encapsulates Bucket Arguments.
- */
-public final class KsmBucketArgs {
- /**
- * Name of the volume in which the bucket belongs to.
- */
- private final String volumeName;
- /**
- * Name of the bucket.
- */
- private final String bucketName;
- /**
- * ACL's that are to be added for the bucket.
- */
- private List<OzoneAcl> addAcls;
- /**
- * ACL's that are to be removed from the bucket.
- */
- private List<OzoneAcl> removeAcls;
- /**
- * Bucket Version flag.
- */
- private Boolean isVersionEnabled;
- /**
- * Type of storage to be used for this bucket.
- * [RAM_DISK, SSD, DISK, ARCHIVE]
- */
- private StorageType storageType;
-
- /**
- * Private constructor, constructed via builder.
- * @param volumeName - Volume name.
- * @param bucketName - Bucket name.
- * @param addAcls - ACL's to be added.
- * @param removeAcls - ACL's to be removed.
- * @param isVersionEnabled - Bucket version flag.
- * @param storageType - Storage type to be used.
- */
- private KsmBucketArgs(String volumeName, String bucketName,
- List<OzoneAcl> addAcls, List<OzoneAcl> removeAcls,
- Boolean isVersionEnabled, StorageType storageType) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.addAcls = addAcls;
- this.removeAcls = removeAcls;
- this.isVersionEnabled = isVersionEnabled;
- this.storageType = storageType;
- }
-
- /**
- * Returns the Volume Name.
- * @return String.
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Returns the Bucket Name.
- * @return String
- */
- public String getBucketName() {
- return bucketName;
- }
-
- /**
- * Returns the ACL's that are to be added.
- * @return List<OzoneAclInfo>
- */
- public List<OzoneAcl> getAddAcls() {
- return addAcls;
- }
-
- /**
- * Returns the ACL's that are to be removed.
- * @return List<OzoneAclInfo>
- */
- public List<OzoneAcl> getRemoveAcls() {
- return removeAcls;
- }
-
- /**
- * Returns true if bucket version is enabled, else false.
- * @return isVersionEnabled
- */
- public Boolean getIsVersionEnabled() {
- return isVersionEnabled;
- }
-
- /**
- * Returns the type of storage to be used.
- * @return StorageType
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Returns new builder class that builds a KsmBucketArgs.
- *
- * @return Builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Builder for KsmBucketArgs.
- */
- public static class Builder {
- private String volumeName;
- private String bucketName;
- private List<OzoneAcl> addAcls;
- private List<OzoneAcl> removeAcls;
- private Boolean isVersionEnabled;
- private StorageType storageType;
-
- public Builder setVolumeName(String volume) {
- this.volumeName = volume;
- return this;
- }
-
- public Builder setBucketName(String bucket) {
- this.bucketName = bucket;
- return this;
- }
-
- public Builder setAddAcls(List<OzoneAcl> acls) {
- this.addAcls = acls;
- return this;
- }
-
- public Builder setRemoveAcls(List<OzoneAcl> acls) {
- this.removeAcls = acls;
- return this;
- }
-
- public Builder setIsVersionEnabled(Boolean versionFlag) {
- this.isVersionEnabled = versionFlag;
- return this;
- }
-
- public Builder setStorageType(StorageType storage) {
- this.storageType = storage;
- return this;
- }
-
- /**
- * Constructs the KsmBucketArgs.
- * @return instance of KsmBucketArgs.
- */
- public KsmBucketArgs build() {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- return new KsmBucketArgs(volumeName, bucketName, addAcls,
- removeAcls, isVersionEnabled, storageType);
- }
- }
-
- /**
- * Creates BucketArgs protobuf from KsmBucketArgs.
- */
- public BucketArgs getProtobuf() {
- BucketArgs.Builder builder = BucketArgs.newBuilder();
- builder.setVolumeName(volumeName)
- .setBucketName(bucketName);
- if(addAcls != null && !addAcls.isEmpty()) {
- builder.addAllAddAcls(addAcls.stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
- }
- if(removeAcls != null && !removeAcls.isEmpty()) {
- builder.addAllRemoveAcls(removeAcls.stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()));
- }
- if(isVersionEnabled != null) {
- builder.setIsVersionEnabled(isVersionEnabled);
- }
- if(storageType != null) {
- builder.setStorageType(
- PBHelperClient.convertStorageType(storageType));
- }
- return builder.build();
- }
-
- /**
- * Parses BucketInfo protobuf and creates KsmBucketArgs.
- * @param bucketArgs
- * @return instance of KsmBucketArgs
- */
- public static KsmBucketArgs getFromProtobuf(BucketArgs bucketArgs) {
- return new KsmBucketArgs(bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- bucketArgs.getAddAclsList().stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
- bucketArgs.getRemoveAclsList().stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
- bucketArgs.hasIsVersionEnabled() ?
- bucketArgs.getIsVersionEnabled() : null,
- bucketArgs.hasStorageType() ? PBHelperClient.convertStorageType(
- bucketArgs.getStorageType()) : null);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
deleted file mode 100644
index a49137a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/helpers/KsmBucketInfo.java
+++ /dev/null
@@ -1,235 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * A class that encapsulates Bucket Info.
- */
-public final class KsmBucketInfo {
- /**
- * Name of the volume in which the bucket belongs to.
- */
- private final String volumeName;
- /**
- * Name of the bucket.
- */
- private final String bucketName;
- /**
- * ACL Information.
- */
- private List<OzoneAcl> acls;
- /**
- * Bucket Version flag.
- */
- private Boolean isVersionEnabled;
- /**
- * Type of storage to be used for this bucket.
- * [RAM_DISK, SSD, DISK, ARCHIVE]
- */
- private StorageType storageType;
- /**
- * Creation time of bucket.
- */
- private final long creationTime;
-
- /**
- * Private constructor, constructed via builder.
- * @param volumeName - Volume name.
- * @param bucketName - Bucket name.
- * @param acls - list of ACLs.
- * @param isVersionEnabled - Bucket version flag.
- * @param storageType - Storage type to be used.
- * @param creationTime - Bucket creation time.
- */
- private KsmBucketInfo(String volumeName, String bucketName,
- List<OzoneAcl> acls, boolean isVersionEnabled,
- StorageType storageType, long creationTime) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.acls = acls;
- this.isVersionEnabled = isVersionEnabled;
- this.storageType = storageType;
- this.creationTime = creationTime;
- }
-
- /**
- * Returns the Volume Name.
- * @return String.
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Returns the Bucket Name.
- * @return String
- */
- public String getBucketName() {
- return bucketName;
- }
-
- /**
- * Returns the ACL's associated with this bucket.
- * @return List<OzoneAcl>
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Returns true if bucket version is enabled, else false.
- * @return isVersionEnabled
- */
- public boolean getIsVersionEnabled() {
- return isVersionEnabled;
- }
-
- /**
- * Returns the type of storage to be used.
- * @return StorageType
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Returns creation time.
- *
- * @return long
- */
- public long getCreationTime() {
- return creationTime;
- }
-
- /**
- * Returns new builder class that builds a KsmBucketInfo.
- *
- * @return Builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Builder for KsmBucketInfo.
- */
- public static class Builder {
- private String volumeName;
- private String bucketName;
- private List<OzoneAcl> acls;
- private Boolean isVersionEnabled;
- private StorageType storageType;
- private long creationTime;
-
- Builder() {
- //Default values
- this.acls = new LinkedList<>();
- this.isVersionEnabled = false;
- this.storageType = StorageType.DISK;
- }
-
- public Builder setVolumeName(String volume) {
- this.volumeName = volume;
- return this;
- }
-
- public Builder setBucketName(String bucket) {
- this.bucketName = bucket;
- return this;
- }
-
- public Builder setAcls(List<OzoneAcl> listOfAcls) {
- this.acls = listOfAcls;
- return this;
- }
-
- public Builder setIsVersionEnabled(Boolean versionFlag) {
- this.isVersionEnabled = versionFlag;
- return this;
- }
-
- public Builder setStorageType(StorageType storage) {
- this.storageType = storage;
- return this;
- }
-
- public Builder setCreationTime(long createdOn) {
- this.creationTime = createdOn;
- return this;
- }
-
- /**
- * Constructs the KsmBucketInfo.
- * @return instance of KsmBucketInfo.
- */
- public KsmBucketInfo build() {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(acls);
- Preconditions.checkNotNull(isVersionEnabled);
- Preconditions.checkNotNull(storageType);
-
- return new KsmBucketInfo(volumeName, bucketName, acls,
- isVersionEnabled, storageType, creationTime);
- }
- }
-
- /**
- * Creates BucketInfo protobuf from KsmBucketInfo.
- */
- public BucketInfo getProtobuf() {
- return BucketInfo.newBuilder()
- .setVolumeName(volumeName)
- .setBucketName(bucketName)
- .addAllAcls(acls.stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()))
- .setIsVersionEnabled(isVersionEnabled)
- .setStorageType(PBHelperClient.convertStorageType(
- storageType))
- .setCreationTime(creationTime)
- .build();
- }
-
- /**
- * Parses BucketInfo protobuf and creates KsmBucketInfo.
- * @param bucketInfo
- * @return instance of KsmBucketInfo
- */
- public static KsmBucketInfo getFromProtobuf(BucketInfo bucketInfo) {
- return new KsmBucketInfo(
- bucketInfo.getVolumeName(),
- bucketInfo.getBucketName(),
- bucketInfo.getAclsList().stream().map(
- KSMPBHelper::convertOzoneAcl).collect(Collectors.toList()),
- bucketInfo.getIsVersionEnabled(),
- PBHelperClient.convertStorageType(
- bucketInfo.getStorageType()), bucketInfo.getCreationTime());
- }
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
deleted file mode 100644
index cfdf025..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/OzoneUtils.java
+++ /dev/null
@@ -1,303 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.utils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.MediaType;
-import java.io.File;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.nio.charset.Charset;
-import java.nio.file.Paths;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.List;
-import java.util.Locale;
-import java.util.TimeZone;
-import java.util.UUID;
-
-/**
- * Set of Utility functions used in ozone.
- */
-@InterfaceAudience.Private
-public final class OzoneUtils {
-
- public static final String ENCODING_NAME = "UTF-8";
- public static final Charset ENCODING = Charset.forName(ENCODING_NAME);
-
- private OzoneUtils() {
- // Never constructed
- }
-
- /**
- * Date format that used in ozone. Here the format is thread safe to use.
- */
- private static final ThreadLocal<SimpleDateFormat> DATE_FORMAT =
- new ThreadLocal<SimpleDateFormat>() {
- @Override
- protected SimpleDateFormat initialValue() {
- SimpleDateFormat format = new SimpleDateFormat(
- OzoneConsts.OZONE_DATE_FORMAT, Locale.US);
- format.setTimeZone(TimeZone.getTimeZone(OzoneConsts.OZONE_TIME_ZONE));
-
- return format;
- }
- };
-
- /**
- * verifies that bucket name / volume name is a valid DNS name.
- *
- * @param resName Bucket or volume Name to be validated
- *
- * @throws IllegalArgumentException
- */
- public static void verifyResourceName(String resName)
- throws IllegalArgumentException {
- OzoneClientUtils.verifyResourceName(resName);
- }
-
- /**
- * Verifies that max key length is a valid value.
- *
- * @param length
- * The max key length to be validated
- *
- * @throws IllegalArgumentException
- */
- public static void verifyMaxKeyLength(String length)
- throws IllegalArgumentException {
- int maxKey = 0;
- try {
- maxKey = Integer.parseInt(length);
- } catch (NumberFormatException nfe) {
- throw new IllegalArgumentException(
- "Invalid max key length, the vaule should be digital.");
- }
-
- if (maxKey <= 0) {
- throw new IllegalArgumentException(
- "Invalid max key length, the vaule should be a positive number.");
- }
- }
-
- /**
- * Returns a random Request ID.
- *
- * Request ID is returned to the client as well as flows through the system
- * facilitating debugging on why a certain request failed.
- *
- * @return String random request ID
- */
- public static String getRequestID() {
- return UUID.randomUUID().toString();
- }
-
- /**
- * Return host name if possible.
- *
- * @return Host Name or localhost
- */
- public static String getHostName() {
- String host = "localhost";
- try {
- host = InetAddress.getLocalHost().getHostName();
- } catch (UnknownHostException e) {
- // Ignore the error
- }
- return host;
- }
-
- /**
- * Basic validate routine to make sure that all the
- * required headers are in place.
- *
- * @param request - http request
- * @param headers - http headers
- * @param reqId - request id
- * @param resource - Resource Name
- * @param hostname - Hostname
- *
- * @throws OzoneException
- */
- public static void validate(Request request, HttpHeaders headers,
- String reqId, String resource, String hostname)
- throws OzoneException {
-
- List<String> ozHeader =
- headers.getRequestHeader(Header.OZONE_VERSION_HEADER);
- if (ozHeader == null) {
- throw ErrorTable
- .newError(ErrorTable.MISSING_VERSION, reqId, resource, hostname);
- }
-
- List<String> date = headers.getRequestHeader(HttpHeaders.DATE);
- if (date == null) {
- throw ErrorTable
- .newError(ErrorTable.MISSING_DATE, reqId, resource, hostname);
- }
-
- /*
- TODO :
- Ignore the results for time being. Eventually we can validate if the
- request Date time is too skewed and reject if it is so.
- */
- parseDate(date.get(0), reqId, resource, hostname);
-
- }
-
- /**
- * Parses the Date String coming from the Users.
- *
- * @param dateString - Date String
- * @param reqID - Ozone Request ID
- * @param resource - Resource Name
- * @param hostname - HostName
- *
- * @return - Date
- *
- * @throws OzoneException - in case of parsing error
- */
- public static synchronized Date parseDate(String dateString, String reqID,
- String resource, String hostname)
- throws OzoneException {
- try {
- return DATE_FORMAT.get().parse(dateString);
- } catch (ParseException ex) {
- OzoneException exp =
- ErrorTable.newError(ErrorTable.BAD_DATE, reqID, resource, hostname);
- exp.setMessage(ex.getMessage());
- throw exp;
- }
- }
-
- /**
- * Returns a response with appropriate OZONE headers and payload.
- *
- * @param args - UserArgs or Inherited class
- * @param statusCode - HttpStatus code
- * @param payload - Content Body
- *
- * @return JAX-RS Response
- */
- public static Response getResponse(UserArgs args, int statusCode,
- String payload) {
- String date = DATE_FORMAT.get().format(new Date(Time.now()));
- return Response.ok(payload)
- .header(Header.OZONE_SERVER_NAME, args.getHostName())
- .header(Header.OZONE_REQUEST_ID, args.getRequestID())
- .header(HttpHeaders.DATE, date).status(statusCode).build();
- }
-
- /**
- * Returns a response with appropriate OZONE headers and payload.
- *
- * @param args - UserArgs or Inherited class
- * @param statusCode - HttpStatus code
- * @param stream InputStream
- *
- * @return JAX-RS Response
- */
- public static Response getResponse(UserArgs args, int statusCode,
- LengthInputStream stream) {
- String date = DATE_FORMAT.get().format(new Date(Time.now()));
- return Response.ok(stream, MediaType.APPLICATION_OCTET_STREAM)
- .header(Header.OZONE_SERVER_NAME, args.getHostName())
- .header(Header.OZONE_REQUEST_ID, args.getRequestID())
- .header(HttpHeaders.DATE, date).status(statusCode)
- .header(HttpHeaders.CONTENT_LENGTH, stream.getLength())
- .build();
-
- }
-
- /**
- * Checks and creates Ozone Metadir Path if it does not exist.
- *
- * @param conf - Configuration
- *
- * @return File MetaDir
- */
- public static File getOzoneMetaDirPath(Configuration conf) {
- String metaDirPath = conf.getTrimmed(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(metaDirPath);
- File dirPath = new File(metaDirPath);
- if (!dirPath.exists() && !dirPath.mkdirs()) {
- throw new IllegalArgumentException("Unable to create paths. Path: " +
- dirPath);
- }
- return dirPath;
- }
-
- /**
- * Get the path for datanode id file.
- *
- * @param conf - Configuration
- * @return the path of datanode id as string
- */
- public static String getDatanodeIDPath(Configuration conf) {
- String dataNodeIDPath = conf.get(ScmConfigKeys.OZONE_SCM_DATANODE_ID);
- if (dataNodeIDPath == null) {
- String metaPath = conf.get(OzoneConfigKeys.OZONE_METADATA_DIRS);
- if (Strings.isNullOrEmpty(metaPath)) {
- // this means meta data is not found, in theory should not happen at
- // this point because should've failed earlier.
- throw new IllegalArgumentException("Unable to locate meta data" +
- "directory when getting datanode id path");
- }
- dataNodeIDPath = Paths.get(metaPath,
- ScmConfigKeys.OZONE_SCM_DATANODE_ID_PATH_DEFAULT).toString();
- }
- return dataNodeIDPath;
- }
-
- /**
- * Convert time in millisecond to a human readable format required in ozone.
- * @return a human readable string for the input time
- */
- public static String formatTime(long millis) {
- return DATE_FORMAT.get().format(millis);
- }
-
- /**
- * Convert time in ozone date format to millisecond.
- * @return time in milliseconds
- */
- public static long formatDate(String date) throws ParseException {
- Preconditions.checkNotNull(date, "Date string should not be null.");
- return DATE_FORMAT.get().parse(date).getTime();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
deleted file mode 100644
index 178157f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.utils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
deleted file mode 100644
index 2ff4e55..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundService.java
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.Lists;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.ScheduledExecutorService;
-import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.CompletionService;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Future;
-import java.util.concurrent.ExecutionException;
-
-/**
- * An abstract class for a background service in ozone.
- * A background service schedules multiple child tasks in parallel
- * in a certain period. In each interval, it waits until all the tasks
- * finish execution and then schedule next interval.
- */
-public abstract class BackgroundService {
-
- @VisibleForTesting
- public static final Logger LOG =
- LoggerFactory.getLogger(BackgroundService.class);
-
- // Executor to launch child tasks
- private final ScheduledExecutorService exec;
- private final ThreadGroup threadGroup;
- private final ThreadFactory threadFactory;
- private final String serviceName;
- private final long interval;
- private final long serviceTimeout;
- private final TimeUnit unit;
- private final PeriodicalTask service;
-
- public BackgroundService(String serviceName, long interval,
- TimeUnit unit, int threadPoolSize, long serviceTimeout) {
- this.interval = interval;
- this.unit = unit;
- this.serviceName = serviceName;
- this.serviceTimeout = serviceTimeout;
- threadGroup = new ThreadGroup(serviceName);
- ThreadFactory tf = r -> new Thread(threadGroup, r);
- threadFactory = new ThreadFactoryBuilder()
- .setThreadFactory(tf)
- .setDaemon(true)
- .setNameFormat(serviceName + "#%d")
- .build();
- exec = Executors.newScheduledThreadPool(threadPoolSize, threadFactory);
- service = new PeriodicalTask();
- }
-
- protected ExecutorService getExecutorService() {
- return this.exec;
- }
-
- @VisibleForTesting
- public int getThreadCount() {
- return threadGroup.activeCount();
- }
-
- @VisibleForTesting
- public void triggerBackgroundTaskForTesting() {
- service.run();
- }
-
- // start service
- public void start() {
- exec.scheduleWithFixedDelay(service, 0, interval, unit);
- }
-
- public abstract BackgroundTaskQueue getTasks();
-
- /**
- * Run one or more background tasks concurrently.
- * Wait until all tasks to return the result.
- */
- public class PeriodicalTask implements Runnable {
- @Override
- public synchronized void run() {
- LOG.debug("Running background service : {}", serviceName);
- BackgroundTaskQueue tasks = getTasks();
- if (tasks.isEmpty()) {
- // No task found, or some problems to init tasks
- // return and retry in next interval.
- return;
- }
-
- LOG.debug("Number of background tasks to execute : {}", tasks.size());
- CompletionService<BackgroundTaskResult> taskCompletionService =
- new ExecutorCompletionService<>(exec);
-
- List<Future<BackgroundTaskResult>> results = Lists.newArrayList();
- while (tasks.size() > 0) {
- BackgroundTask task = tasks.poll();
- Future<BackgroundTaskResult> result =
- taskCompletionService.submit(task);
- results.add(result);
- }
-
- results.parallelStream().forEach(taskResultFuture -> {
- try {
- // Collect task results
- BackgroundTaskResult result = serviceTimeout > 0
- ? taskResultFuture.get(serviceTimeout, TimeUnit.MILLISECONDS)
- : taskResultFuture.get();
- if (LOG.isDebugEnabled()) {
- LOG.debug("task execution result size {}", result.getSize());
- }
- } catch (InterruptedException | ExecutionException e) {
- LOG.warn(
- "Background task fails to execute, "
- + "retrying in next interval", e);
- } catch (TimeoutException e) {
- LOG.warn("Background task executes timed out, "
- + "retrying in next interval", e);
- }
- });
- }
- }
-
- // shutdown and make sure all threads are properly released.
- public void shutdown() {
- LOG.info("Shutting down service {}", this.serviceName);
- exec.shutdown();
- try {
- if (!exec.awaitTermination(60, TimeUnit.SECONDS)) {
- exec.shutdownNow();
- }
- } catch (InterruptedException e) {
- exec.shutdownNow();
- }
- if (threadGroup.activeCount() == 0 && !threadGroup.isDestroyed()) {
- threadGroup.destroy();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
deleted file mode 100644
index 47e8ebc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTask.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.util.concurrent.Callable;
-
-/**
- * A task thread to run by {@link BackgroundService}.
- */
-public interface BackgroundTask<T> extends Callable<T> {
-
- int getPriority();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
deleted file mode 100644
index b56ef0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskQueue.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.util.PriorityQueue;
-
-/**
- * A priority queue that stores a number of {@link BackgroundTask}.
- */
-public class BackgroundTaskQueue {
-
- private final PriorityQueue<BackgroundTask> tasks;
-
- public BackgroundTaskQueue() {
- tasks = new PriorityQueue<>((task1, task2)
- -> task1.getPriority() - task2.getPriority());
- }
-
- /**
- * @return the head task in this queue.
- */
- public synchronized BackgroundTask poll() {
- return tasks.poll();
- }
-
- /**
- * Add a {@link BackgroundTask} to the queue,
- * the task will be sorted by its priority.
- *
- * @param task
- */
- public synchronized void add(BackgroundTask task) {
- tasks.add(task);
- }
-
- /**
- * @return true if the queue contains no task, false otherwise.
- */
- public synchronized boolean isEmpty() {
- return tasks.isEmpty();
- }
-
- /**
- * @return the size of the queue.
- */
- public synchronized int size() {
- return tasks.size();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
deleted file mode 100644
index 198300f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BackgroundTaskResult.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.utils;
-
-/**
- * Result of a {@link BackgroundTask}.
- */
-public interface BackgroundTaskResult {
-
- /**
- * Returns the size of entries included in this result.
- */
- int getSize();
-
- /**
- * An empty task result implementation.
- */
- class EmptyTaskResult implements BackgroundTaskResult {
-
- public static EmptyTaskResult newResult() {
- return new EmptyTaskResult();
- }
-
- @Override
- public int getSize() {
- return 0;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
deleted file mode 100644
index 47699eb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/BatchOperation.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.collect.Lists;
-
-import java.util.List;
-
-/**
- * An utility class to store a batch of DB write operations.
- */
-public class BatchOperation {
-
- /**
- * Enum for write operations.
- */
- public enum Operation {
- DELETE, PUT
- }
-
- private List<SingleOperation> operations =
- Lists.newArrayList();
-
- /**
- * Add a PUT operation into the batch.
- */
- public void put(byte[] key, byte[] value) {
- operations.add(new SingleOperation(Operation.PUT, key, value));
- }
-
- /**
- * Add a DELETE operation into the batch.
- */
- public void delete(byte[] key) {
- operations.add(new SingleOperation(Operation.DELETE, key, null));
-
- }
-
- public List<SingleOperation> getOperations() {
- return operations;
- }
-
- /**
- * A SingleOperation represents a PUT or DELETE operation
- * and the data the operation needs to manipulates.
- */
- public static class SingleOperation {
-
- private Operation opt;
- private byte[] key;
- private byte[] value;
-
- public SingleOperation(Operation opt, byte[] key, byte[] value) {
- this.opt = opt;
- if (key == null) {
- throw new IllegalArgumentException("key cannot be null");
- }
- this.key = key.clone();
- this.value = value == null ? null : value.clone();
- }
-
- public Operation getOpt() {
- return opt;
- }
-
- public byte[] getKey() {
- return key.clone();
- }
-
- public byte[] getValue() {
- return value == null ? null : value.clone();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
deleted file mode 100644
index c407398..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/EntryConsumer.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import java.io.IOException;
-
-/**
- * A consumer for metadata store key-value entries.
- * Used by {@link MetadataStore} class.
- */
-@FunctionalInterface
-public interface EntryConsumer {
-
- /**
- * Consumes a key and value and produces a boolean result.
- * @param key key
- * @param value value
- * @return a boolean value produced by the consumer
- * @throws IOException
- */
- boolean consume(byte[] key, byte[] value) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
deleted file mode 100644
index 38cee76..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/LevelDBStore.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.fusesource.leveldbjni.JniDBFactory;
-import org.iq80.leveldb.WriteBatch;
-import org.iq80.leveldb.DB;
-import org.iq80.leveldb.Options;
-import org.iq80.leveldb.WriteOptions;
-import org.iq80.leveldb.DBIterator;
-import org.iq80.leveldb.Snapshot;
-import org.iq80.leveldb.ReadOptions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * LevelDB interface.
- */
-public class LevelDBStore implements MetadataStore {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(LevelDBStore.class);
-
- private DB db;
- private final File dbFile;
- private final Options dbOptions;
- private final WriteOptions writeOptions;
-
- public LevelDBStore(File dbPath, boolean createIfMissing)
- throws IOException {
- dbOptions = new Options();
- dbOptions.createIfMissing(createIfMissing);
- this.dbFile = dbPath;
- this.writeOptions = new WriteOptions().sync(true);
- openDB(dbPath, dbOptions);
- }
-
- /**
- * Opens a DB file.
- *
- * @param dbPath - DB File path
- * @throws IOException
- */
- public LevelDBStore(File dbPath, Options options)
- throws IOException {
- dbOptions = options;
- this.dbFile = dbPath;
- this.writeOptions = new WriteOptions().sync(true);
- openDB(dbPath, dbOptions);
- }
-
- private void openDB(File dbPath, Options options) throws IOException {
- db = JniDBFactory.factory.open(dbPath, options);
- if (LOG.isDebugEnabled()) {
- LOG.debug("LevelDB successfully opened");
- LOG.debug("[Option] cacheSize = " + options.cacheSize());
- LOG.debug("[Option] createIfMissing = " + options.createIfMissing());
- LOG.debug("[Option] blockSize = " + options.blockSize());
- LOG.debug("[Option] compressionType= " + options.compressionType());
- LOG.debug("[Option] maxOpenFiles= " + options.maxOpenFiles());
- LOG.debug("[Option] writeBufferSize= "+ options.writeBufferSize());
- }
- }
-
- /**
- * Puts a Key into file.
- *
- * @param key - key
- * @param value - value
- */
- @Override
- public void put(byte[] key, byte[] value) {
- db.put(key, value, writeOptions);
- }
-
- /**
- * Get Key.
- *
- * @param key key
- * @return value
- */
- @Override
- public byte[] get(byte[] key) {
- return db.get(key);
- }
-
- /**
- * Delete Key.
- *
- * @param key - Key
- */
- @Override
- public void delete(byte[] key) {
- db.delete(key);
- }
-
- /**
- * Closes the DB.
- *
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- if (db != null){
- db.close();
- }
- }
-
- /**
- * Returns true if the DB is empty.
- *
- * @return boolean
- * @throws IOException
- */
- @Override
- public boolean isEmpty() throws IOException {
- try (DBIterator iter = db.iterator()) {
- iter.seekToFirst();
- boolean hasNext = !iter.hasNext();
- return hasNext;
- }
- }
-
- /**
- * Returns the actual levelDB object.
- * @return DB handle.
- */
- public DB getDB() {
- return db;
- }
-
- /**
- * Returns an iterator on all the key-value pairs in the DB.
- * @return an iterator on DB entries.
- */
- public DBIterator getIterator() {
- return db.iterator();
- }
-
-
- @Override
- public void destroy() throws IOException {
- close();
- JniDBFactory.factory.destroy(dbFile, dbOptions);
- }
-
- @Override
- public ImmutablePair<byte[], byte[]> peekAround(int offset,
- byte[] from) throws IOException, IllegalArgumentException {
- try (DBIterator it = db.iterator()) {
- if (from == null) {
- it.seekToFirst();
- } else {
- it.seek(from);
- }
- if (!it.hasNext()) {
- return null;
- }
- switch (offset) {
- case 0:
- Entry<byte[], byte[]> current = it.next();
- return new ImmutablePair<>(current.getKey(), current.getValue());
- case 1:
- if (it.next() != null && it.hasNext()) {
- Entry<byte[], byte[]> next = it.peekNext();
- return new ImmutablePair<>(next.getKey(), next.getValue());
- }
- break;
- case -1:
- if (it.hasPrev()) {
- Entry<byte[], byte[]> prev = it.peekPrev();
- return new ImmutablePair<>(prev.getKey(), prev.getValue());
- }
- break;
- default:
- throw new IllegalArgumentException(
- "Position can only be -1, 0 " + "or 1, but found " + offset);
- }
- }
- return null;
- }
-
- @Override
- public void iterate(byte[] from, EntryConsumer consumer)
- throws IOException {
- try (DBIterator iter = db.iterator()) {
- if (from != null) {
- iter.seek(from);
- } else {
- iter.seekToFirst();
- }
- while (iter.hasNext()) {
- Entry<byte[], byte[]> current = iter.next();
- if (!consumer.consume(current.getKey(),
- current.getValue())) {
- break;
- }
- }
- }
- }
-
- /**
- * Compacts the DB by removing deleted keys etc.
- * @throws IOException if there is an error.
- */
- @Override
- public void compactDB() throws IOException {
- if(db != null) {
- // From LevelDB docs : begin == null and end == null means the whole DB.
- db.compactRange(null, null);
- }
- }
-
- @Override
- public void writeBatch(BatchOperation operation) throws IOException {
- List<BatchOperation.SingleOperation> operations =
- operation.getOperations();
- if (!operations.isEmpty()) {
- try (WriteBatch writeBatch = db.createWriteBatch()) {
- for (BatchOperation.SingleOperation opt : operations) {
- switch (opt.getOpt()) {
- case DELETE:
- writeBatch.delete(opt.getKey());
- break;
- case PUT:
- writeBatch.put(opt.getKey(), opt.getValue());
- break;
- default:
- throw new IllegalArgumentException("Invalid operation "
- + opt.getOpt());
- }
- }
- db.write(writeBatch);
- }
- }
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, false, filters);
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, true, filters);
- }
-
- /**
- * Returns a certain range of key value pairs as a list based on a
- * startKey or count. Further a {@link MetadataKeyFilter} can be added to
- * filter keys if necessary. To prevent race conditions while listing
- * entries, this implementation takes a snapshot and lists the entries from
- * the snapshot. This may, on the other hand, cause the range result slight
- * different with actual data if data is updating concurrently.
- * <p>
- * If the startKey is specified and found in levelDB, this key and the keys
- * after this key will be included in the result. If the startKey is null
- * all entries will be included as long as other conditions are satisfied.
- * If the given startKey doesn't exist, an empty list will be returned.
- * <p>
- * The count argument is to limit number of total entries to return,
- * the value for count must be an integer greater than 0.
- * <p>
- * This method allows to specify one or more {@link MetadataKeyFilter}
- * to filter keys by certain condition. Once given, only the entries
- * whose key passes all the filters will be included in the result.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database or an empty list if the
- * startKey is invalid.
- * @throws IOException if there are I/O errors.
- * @throws IllegalArgumentException if count is less than 0.
- */
- private List<Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, boolean sequential, MetadataKeyFilter... filters)
- throws IOException {
- List<Entry<byte[], byte[]>> result = new ArrayList<>();
- long start = System.currentTimeMillis();
- if (count < 0) {
- throw new IllegalArgumentException(
- "Invalid count given " + count + ", count must be greater than 0");
- }
- Snapshot snapShot = null;
- DBIterator dbIter = null;
- try {
- snapShot = db.getSnapshot();
- ReadOptions readOptions = new ReadOptions().snapshot(snapShot);
- dbIter = db.iterator(readOptions);
- if (startKey == null) {
- dbIter.seekToFirst();
- } else {
- if (db.get(startKey) == null) {
- // Key not found, return empty list
- return result;
- }
- dbIter.seek(startKey);
- }
- while (dbIter.hasNext() && result.size() < count) {
- byte[] preKey = dbIter.hasPrev() ? dbIter.peekPrev().getKey() : null;
- byte[] nextKey = dbIter.hasNext() ? dbIter.peekNext().getKey() : null;
- Entry<byte[], byte[]> current = dbIter.next();
-
- if (filters == null) {
- result.add(current);
- } else {
- if (Arrays.asList(filters).stream().allMatch(
- entry -> entry.filterKey(preKey, current.getKey(), nextKey))) {
- result.add(current);
- } else {
- if (result.size() > 0 && sequential) {
- // if the caller asks for a sequential range of results,
- // and we met a dis-match, abort iteration from here.
- // if result is empty, we continue to look for the first match.
- break;
- }
- }
- }
- }
- } finally {
- if (snapShot != null) {
- snapShot.close();
- }
- if (dbIter != null) {
- dbIter.close();
- }
- if (LOG.isDebugEnabled()) {
- if (filters != null) {
- for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
- int scanned = filter.getKeysScannedNum();
- int hinted = filter.getKeysHintedNum();
- if (scanned > 0 || hinted > 0) {
- LOG.debug(
- "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
- filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
- filter.getKeysHintedNum());
- }
- }
- }
- long end = System.currentTimeMillis();
- long timeConsumed = end - start;
- LOG.debug("Time consumed for getRangeKVs() is {}ms,"
- + " result length is {}.", timeConsumed, result.size());
- }
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
deleted file mode 100644
index 3ff0a94..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataKeyFilters.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.utils;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-
-/**
- * An utility class to filter levelDB keys.
- */
-public final class MetadataKeyFilters {
-
- private static KeyPrefixFilter deletingKeyFilter =
- new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX);
-
- private static KeyPrefixFilter normalKeyFilter =
- new MetadataKeyFilters.KeyPrefixFilter(OzoneConsts.DELETING_KEY_PREFIX,
- true);
-
- private MetadataKeyFilters() {
- }
-
- public static KeyPrefixFilter getDeletingKeyFilter() {
- return deletingKeyFilter;
- }
-
- public static KeyPrefixFilter getNormalKeyFilter() {
- return normalKeyFilter;
- }
- /**
- * Interface for levelDB key filters.
- */
- public interface MetadataKeyFilter {
- /**
- * Filter levelDB key with a certain condition.
- *
- * @param preKey previous key.
- * @param currentKey current key.
- * @param nextKey next key.
- * @return true if a certain condition satisfied, return false otherwise.
- */
- boolean filterKey(byte[] preKey, byte[] currentKey, byte[] nextKey);
-
- default int getKeysScannedNum() {
- return 0;
- }
-
- default int getKeysHintedNum() {
- return 0;
- }
- }
-
- /**
- * Utility class to filter key by a string prefix. This filter
- * assumes keys can be parsed to a string.
- */
- public static class KeyPrefixFilter implements MetadataKeyFilter {
-
- private String keyPrefix = null;
- private int keysScanned = 0;
- private int keysHinted = 0;
- private Boolean negative;
-
- public KeyPrefixFilter(String keyPrefix) {
- this(keyPrefix, false);
- }
-
- public KeyPrefixFilter(String keyPrefix, boolean negative) {
- this.keyPrefix = keyPrefix;
- this.negative = negative;
- }
-
- @Override
- public boolean filterKey(byte[] preKey, byte[] currentKey,
- byte[] nextKey) {
- keysScanned++;
- boolean accept = false;
- if (Strings.isNullOrEmpty(keyPrefix)) {
- accept = true;
- } else {
- if (currentKey != null &&
- DFSUtil.bytes2String(currentKey).startsWith(keyPrefix)) {
- keysHinted++;
- accept = true;
- } else {
- accept = false;
- }
- }
- return (negative) ? !accept : accept;
- }
-
- @Override
- public int getKeysScannedNum() {
- return keysScanned;
- }
-
- @Override
- public int getKeysHintedNum() {
- return keysHinted;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
deleted file mode 100644
index b90b08f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStore.java
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-
-/**
- * Interface for key-value store that stores ozone metadata.
- * Ozone metadata is stored as key value pairs, both key and value
- * are arbitrary byte arrays.
- */
-@InterfaceStability.Evolving
-public interface MetadataStore extends Closeable{
-
- /**
- * Puts a key-value pair into the store.
- *
- * @param key metadata key
- * @param value metadata value
- */
- void put(byte[] key, byte[] value) throws IOException;
-
- /**
- * @return true if the metadata store is empty.
- *
- * @throws IOException
- */
- boolean isEmpty() throws IOException;
-
- /**
- * Returns the value mapped to the given key in byte array.
- *
- * @param key metadata key
- * @return value in byte array
- * @throws IOException
- */
- byte[] get(byte[] key) throws IOException;
-
- /**
- * Deletes a key from the metadata store.
- *
- * @param key metadata key
- * @throws IOException
- */
- void delete(byte[] key) throws IOException;
-
- /**
- * Returns a certain range of key value pairs as a list based on a
- * startKey or count. Further a {@link MetadataKeyFilter} can be added to
- * filter keys if necessary. To prevent race conditions while listing
- * entries, this implementation takes a snapshot and lists the entries from
- * the snapshot. This may, on the other hand, cause the range result slight
- * different with actual data if data is updating concurrently.
- * <p>
- * If the startKey is specified and found in levelDB, this key and the keys
- * after this key will be included in the result. If the startKey is null
- * all entries will be included as long as other conditions are satisfied.
- * If the given startKey doesn't exist and empty list will be returned.
- * <p>
- * The count argument is to limit number of total entries to return,
- * the value for count must be an integer greater than 0.
- * <p>
- * This method allows to specify one or more {@link MetadataKeyFilter}
- * to filter keys by certain condition. Once given, only the entries
- * whose key passes all the filters will be included in the result.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database or an empty list if the
- * startKey is invalid.
- * @throws IOException if there are I/O errors.
- * @throws IllegalArgumentException if count is less than 0.
- */
- List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException;
-
- /**
- * This method is very similar to {@link #getRangeKVs}, the only
- * different is this method is supposed to return a sequential range
- * of elements based on the filters. While iterating the elements,
- * if it met any entry that cannot pass the filter, the iterator will stop
- * from this point without looking for next match. If no filter is given,
- * this method behaves just like {@link #getRangeKVs}.
- *
- * @param startKey a start key.
- * @param count max number of entries to return.
- * @param filters customized one or more {@link MetadataKeyFilter}.
- * @return a list of entries found in the database.
- * @throws IOException
- * @throws IllegalArgumentException
- */
- List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException;
-
- /**
- * A batch of PUT, DELETE operations handled as a single atomic write.
- *
- * @throws IOException write fails
- */
- void writeBatch(BatchOperation operation) throws IOException;
-
- /**
- * Compact the entire database.
- * @throws IOException
- */
- void compactDB() throws IOException;
-
- /**
- * Destroy the content of the specified database,
- * a destroyed database will not be able to load again.
- * Be very careful with this method.
- *
- * @throws IOException if I/O error happens
- */
- void destroy() throws IOException;
-
- /**
- * Seek the database to a certain key, returns the key-value
- * pairs around this key based on the given offset. Note, this method
- * can only support offset -1 (left), 0 (current) and 1 (right),
- * any other offset given will cause a {@link IllegalArgumentException}.
- *
- * @param offset offset to the key
- * @param from from which key
- * @return a key-value pair
- * @throws IOException
- */
- ImmutablePair<byte[], byte[]> peekAround(int offset, byte[] from)
- throws IOException, IllegalArgumentException;
-
- /**
- * Iterates entries in the database from a certain key.
- * Applies the given {@link EntryConsumer} to the key and value of
- * each entry, the function produces a boolean result which is used
- * as the criteria to exit from iteration.
- *
- * @param from the start key
- * @param consumer
- * a {@link EntryConsumer} applied to each key and value. If the consumer
- * returns true, continues the iteration to next entry; otherwise exits
- * the iteration.
- * @throws IOException
- */
- void iterate(byte[] from, EntryConsumer consumer)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
deleted file mode 100644
index 095e718..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/MetadataStoreBuilder.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_METADATA_STORE_ROCKSDB_STATISTICS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF;
-import org.iq80.leveldb.Options;
-import org.rocksdb.BlockBasedTableConfig;
-
-import java.io.File;
-import java.io.IOException;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_LEVELDB;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_ROCKSDB;
-
-import org.rocksdb.Statistics;
-import org.rocksdb.StatsLevel;
-
-/**
- * Builder for metadata store.
- */
-public class MetadataStoreBuilder {
-
- private File dbFile;
- private long cacheSize;
- private boolean createIfMissing = true;
- private Configuration conf;
-
- public static MetadataStoreBuilder newBuilder() {
- return new MetadataStoreBuilder();
- }
-
- public MetadataStoreBuilder setDbFile(File dbPath) {
- this.dbFile = dbPath;
- return this;
- }
-
- public MetadataStoreBuilder setCacheSize(long cache) {
- this.cacheSize = cache;
- return this;
- }
-
- public MetadataStoreBuilder setCreateIfMissing(boolean doCreate) {
- this.createIfMissing = doCreate;
- return this;
- }
-
- public MetadataStoreBuilder setConf(Configuration configuration) {
- this.conf = configuration;
- return this;
- }
-
- public MetadataStore build() throws IOException {
- if (dbFile == null) {
- throw new IllegalArgumentException("Failed to build metadata store, "
- + "dbFile is required but not found");
- }
-
- // Build db store based on configuration
- MetadataStore store = null;
- String impl = conf == null ?
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT :
- conf.getTrimmed(OzoneConfigKeys.OZONE_METADATA_STORE_IMPL,
- OzoneConfigKeys.OZONE_METADATA_STORE_IMPL_DEFAULT);
- if (OZONE_METADATA_STORE_IMPL_LEVELDB.equals(impl)) {
- Options options = new Options();
- options.createIfMissing(createIfMissing);
- if (cacheSize > 0) {
- options.cacheSize(cacheSize);
- }
- store = new LevelDBStore(dbFile, options);
- } else if (OZONE_METADATA_STORE_IMPL_ROCKSDB.equals(impl)) {
- org.rocksdb.Options opts = new org.rocksdb.Options();
- opts.setCreateIfMissing(createIfMissing);
-
- if (cacheSize > 0) {
- BlockBasedTableConfig tableConfig = new BlockBasedTableConfig();
- tableConfig.setBlockCacheSize(cacheSize);
- opts.setTableFormatConfig(tableConfig);
- }
-
- String rocksDbStat = conf == null ?
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT :
- conf.getTrimmed(OZONE_METADATA_STORE_ROCKSDB_STATISTICS,
- OZONE_METADATA_STORE_ROCKSDB_STATISTICS_DEFAULT);
-
- if (!rocksDbStat.equals(OZONE_METADATA_STORE_ROCKSDB_STATISTICS_OFF)) {
- Statistics statistics = new Statistics();
- statistics.setStatsLevel(StatsLevel.valueOf(rocksDbStat));
- opts = opts.setStatistics(statistics);
-
- }
- store = new RocksDBStore(dbFile, opts);
- } else {
- throw new IllegalArgumentException("Invalid argument for "
- + OzoneConfigKeys.OZONE_METADATA_STORE_IMPL
- + ". Expecting " + OZONE_METADATA_STORE_IMPL_LEVELDB
- + " or " + OZONE_METADATA_STORE_IMPL_ROCKSDB
- + ", but met " + impl);
- }
- return store;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
deleted file mode 100644
index 2f340a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStore.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
-import org.rocksdb.RocksIterator;
-import org.rocksdb.Options;
-import org.rocksdb.WriteOptions;
-import org.rocksdb.RocksDB;
-import org.rocksdb.RocksDBException;
-import org.rocksdb.WriteBatch;
-import org.rocksdb.DbPath;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.File;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.AbstractMap;
-
-/**
- * RocksDB implementation of ozone metadata store.
- */
-public class RocksDBStore implements MetadataStore {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(RocksDBStore.class);
-
- private RocksDB db = null;
- private File dbLocation;
- private WriteOptions writeOptions;
- private Options dbOptions;
- private ObjectName statMBeanName;
-
- public RocksDBStore(File dbFile, Options options)
- throws IOException {
- Preconditions.checkNotNull(dbFile, "DB file location cannot be null");
- RocksDB.loadLibrary();
- dbOptions = options;
- dbLocation = dbFile;
- writeOptions = new WriteOptions();
- try {
-
- db = RocksDB.open(dbOptions, dbLocation.getAbsolutePath());
- if (dbOptions.statistics() != null) {
-
- Map<String, String> jmxProperties = new HashMap<String, String>();
- jmxProperties.put("dbName", dbFile.getName());
- statMBeanName = MBeans.register("Ozone", "RocksDbStore", jmxProperties,
- new RocksDBStoreMBean(dbOptions.statistics()));
- if (statMBeanName == null) {
- LOG.warn("jmx registration failed during RocksDB init, db path :{}",
- dbFile.getAbsolutePath());
- }
- }
- } catch (RocksDBException e) {
- throw new IOException(
- "Failed init RocksDB, db path : " + dbFile.getAbsolutePath(), e);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("RocksDB successfully opened.");
- LOG.debug("[Option] dbLocation= {}", dbLocation.getAbsolutePath());
- LOG.debug("[Option] createIfMissing = {}", options.createIfMissing());
- LOG.debug("[Option] compactionPriority= {}", options.compactionStyle());
- LOG.debug("[Option] compressionType= {}", options.compressionType());
- LOG.debug("[Option] maxOpenFiles= {}", options.maxOpenFiles());
- LOG.debug("[Option] writeBufferSize= {}", options.writeBufferSize());
- }
- }
-
- private IOException toIOException(String msg, RocksDBException e) {
- String statusCode = e.getStatus() == null ? "N/A" :
- e.getStatus().getCodeString();
- String errMessage = e.getMessage() == null ? "Unknown error" :
- e.getMessage();
- String output = msg + "; status : " + statusCode
- + "; message : " + errMessage;
- return new IOException(output, e);
- }
-
- @Override
- public void put(byte[] key, byte[] value) throws IOException {
- try {
- db.put(writeOptions, key, value);
- } catch (RocksDBException e) {
- throw toIOException("Failed to put key-value to metadata store", e);
- }
- }
-
- @Override
- public boolean isEmpty() throws IOException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- it.seekToFirst();
- return !it.isValid();
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public byte[] get(byte[] key) throws IOException {
- try {
- return db.get(key);
- } catch (RocksDBException e) {
- throw toIOException("Failed to get the value for the given key", e);
- }
- }
-
- @Override
- public void delete(byte[] key) throws IOException {
- try {
- db.delete(key);
- } catch (RocksDBException e) {
- throw toIOException("Failed to delete the given key", e);
- }
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, false, filters);
- }
-
- @Override
- public List<Map.Entry<byte[], byte[]>> getSequentialRangeKVs(byte[] startKey,
- int count, MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- return getRangeKVs(startKey, count, true, filters);
- }
-
- private List<Map.Entry<byte[], byte[]>> getRangeKVs(byte[] startKey,
- int count, boolean sequential,
- MetadataKeyFilters.MetadataKeyFilter... filters)
- throws IOException, IllegalArgumentException {
- List<Map.Entry<byte[], byte[]>> result = new ArrayList<>();
- long start = System.currentTimeMillis();
- if (count < 0) {
- throw new IllegalArgumentException(
- "Invalid count given " + count + ", count must be greater than 0");
- }
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (startKey == null) {
- it.seekToFirst();
- } else {
- if(get(startKey) == null) {
- // Key not found, return empty list
- return result;
- }
- it.seek(startKey);
- }
- while(it.isValid() && result.size() < count) {
- byte[] currentKey = it.key();
- byte[] currentValue = it.value();
-
- it.prev();
- final byte[] prevKey = it.isValid() ? it.key() : null;
-
- it.seek(currentKey);
- it.next();
- final byte[] nextKey = it.isValid() ? it.key() : null;
-
- if (filters == null) {
- result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
- currentValue));
- } else {
- if (Arrays.asList(filters).stream()
- .allMatch(entry -> entry.filterKey(prevKey,
- currentKey, nextKey))) {
- result.add(new AbstractMap.SimpleImmutableEntry<>(currentKey,
- currentValue));
- } else {
- if (result.size() > 0 && sequential) {
- // if the caller asks for a sequential range of results,
- // and we met a dis-match, abort iteration from here.
- // if result is empty, we continue to look for the first match.
- break;
- }
- }
- }
- }
- } finally {
- if (it != null) {
- it.close();
- }
- long end = System.currentTimeMillis();
- long timeConsumed = end - start;
- if (LOG.isDebugEnabled()) {
- if (filters != null) {
- for (MetadataKeyFilters.MetadataKeyFilter filter : filters) {
- int scanned = filter.getKeysScannedNum();
- int hinted = filter.getKeysHintedNum();
- if (scanned > 0 || hinted > 0) {
- LOG.debug(
- "getRangeKVs ({}) numOfKeysScanned={}, numOfKeysHinted={}",
- filter.getClass().getSimpleName(), filter.getKeysScannedNum(),
- filter.getKeysHintedNum());
- }
- }
- }
- LOG.debug("Time consumed for getRangeKVs() is {}ms,"
- + " result length is {}.", timeConsumed, result.size());
- }
- }
- return result;
- }
-
- @Override
- public void writeBatch(BatchOperation operation)
- throws IOException {
- List<BatchOperation.SingleOperation> operations =
- operation.getOperations();
- if (!operations.isEmpty()) {
- try (WriteBatch writeBatch = new WriteBatch()) {
- for (BatchOperation.SingleOperation opt : operations) {
- switch (opt.getOpt()) {
- case DELETE:
- writeBatch.remove(opt.getKey());
- break;
- case PUT:
- writeBatch.put(opt.getKey(), opt.getValue());
- break;
- default:
- throw new IllegalArgumentException("Invalid operation "
- + opt.getOpt());
- }
- }
- db.write(writeOptions, writeBatch);
- } catch (RocksDBException e) {
- throw toIOException("Batch write operation failed", e);
- }
- }
- }
-
- @Override
- public void compactDB() throws IOException {
- if (db != null) {
- try {
- db.compactRange();
- } catch (RocksDBException e) {
- throw toIOException("Failed to compact db", e);
- }
- }
- }
-
- private void deleteQuietly(File fileOrDir) {
- if (fileOrDir != null && fileOrDir.exists()) {
- try {
- FileUtils.forceDelete(fileOrDir);
- } catch (IOException e) {
- LOG.warn("Failed to delete dir {}", fileOrDir.getAbsolutePath(), e);
- }
- }
- }
-
- @Override
- public void destroy() throws IOException {
- // Make sure db is closed.
- close();
-
- // There is no destroydb java API available,
- // equivalently we can delete all db directories.
- deleteQuietly(dbLocation);
- deleteQuietly(new File(dbOptions.dbLogDir()));
- deleteQuietly(new File(dbOptions.walDir()));
- List<DbPath> dbPaths = dbOptions.dbPaths();
- if (dbPaths != null) {
- dbPaths.forEach(dbPath -> {
- deleteQuietly(new File(dbPath.toString()));
- });
- }
- }
-
- @Override
- public ImmutablePair<byte[], byte[]> peekAround(int offset,
- byte[] from) throws IOException, IllegalArgumentException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (from == null) {
- it.seekToFirst();
- } else {
- it.seek(from);
- }
- if (!it.isValid()) {
- return null;
- }
-
- switch (offset) {
- case 0:
- break;
- case 1:
- it.next();
- break;
- case -1:
- it.prev();
- break;
- default:
- throw new IllegalArgumentException(
- "Position can only be -1, 0 " + "or 1, but found " + offset);
- }
- return it.isValid() ? new ImmutablePair<>(it.key(), it.value()) : null;
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public void iterate(byte[] from, EntryConsumer consumer)
- throws IOException {
- RocksIterator it = null;
- try {
- it = db.newIterator();
- if (from != null) {
- it.seek(from);
- } else {
- it.seekToFirst();
- }
- while (it.isValid()) {
- if (!consumer.consume(it.key(), it.value())) {
- break;
- }
- it.next();
- }
- } finally {
- if (it != null) {
- it.close();
- }
- }
- }
-
- @Override
- public void close() throws IOException {
- if (statMBeanName != null) {
- MBeans.unregister(statMBeanName);
- }
- if (db != null) {
- db.close();
- }
-
- }
-
- @VisibleForTesting
- protected ObjectName getStatMBeanName() {
- return statMBeanName;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
deleted file mode 100644
index df0f41c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/RocksDBStoreMBean.java
+++ /dev/null
@@ -1,148 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.utils;
-
-import org.rocksdb.HistogramData;
-import org.rocksdb.HistogramType;
-import org.rocksdb.Statistics;
-import org.rocksdb.TickerType;
-
-import javax.management.Attribute;
-import javax.management.AttributeList;
-import javax.management.AttributeNotFoundException;
-import javax.management.DynamicMBean;
-import javax.management.InvalidAttributeValueException;
-import javax.management.MBeanAttributeInfo;
-import javax.management.MBeanException;
-import javax.management.MBeanInfo;
-import javax.management.ReflectionException;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Stream;
-
-/**
- * Adapter JMX bean to publish all the Rocksdb metrics.
- */
-public class RocksDBStoreMBean implements DynamicMBean {
-
- private Statistics statistics;
-
- private Set<String> histogramAttributes = new HashSet<>();
-
- public RocksDBStoreMBean(Statistics statistics) {
- this.statistics = statistics;
- histogramAttributes.add("Average");
- histogramAttributes.add("Median");
- histogramAttributes.add("Percentile95");
- histogramAttributes.add("Percentile99");
- histogramAttributes.add("StandardDeviation");
- }
-
- @Override
- public Object getAttribute(String attribute)
- throws AttributeNotFoundException, MBeanException, ReflectionException {
- for (String histogramAttribute : histogramAttributes) {
- if (attribute.endsWith("_" + histogramAttribute.toUpperCase())) {
- String keyName = attribute
- .substring(0, attribute.length() - histogramAttribute.length() -1);
- try {
- HistogramData histogram =
- statistics.getHistogramData(HistogramType.valueOf(keyName));
- try {
- Method method =
- HistogramData.class.getMethod("get" + histogramAttribute);
- return method.invoke(histogram);
- } catch (Exception e) {
- throw new ReflectionException(e,
- "Can't read attribute " + attribute);
- }
- } catch (IllegalArgumentException exception) {
- throw new AttributeNotFoundException(
- "No such attribute in RocksDB stats: " + attribute);
- }
- }
- }
- try {
- return statistics.getTickerCount(TickerType.valueOf(attribute));
- } catch (IllegalArgumentException ex) {
- throw new AttributeNotFoundException(
- "No such attribute in RocksDB stats: " + attribute);
- }
- }
-
- @Override
- public void setAttribute(Attribute attribute)
- throws AttributeNotFoundException, InvalidAttributeValueException,
- MBeanException, ReflectionException {
-
- }
-
- @Override
- public AttributeList getAttributes(String[] attributes) {
- AttributeList result = new AttributeList();
- for (String attributeName : attributes) {
- try {
- Object value = getAttribute(attributeName);
- result.add(value);
- } catch (Exception e) {
- //TODO
- }
- }
- return result;
- }
-
- @Override
- public AttributeList setAttributes(AttributeList attributes) {
- return null;
- }
-
- @Override
- public Object invoke(String actionName, Object[] params, String[] signature)
- throws MBeanException, ReflectionException {
- return null;
- }
-
- @Override
- public MBeanInfo getMBeanInfo() {
-
- List<MBeanAttributeInfo> attributes = new ArrayList<>();
- for (TickerType tickerType : TickerType.values()) {
- attributes.add(new MBeanAttributeInfo(tickerType.name(), "long",
- "RocksDBStat: " + tickerType.name(), true, false, false));
- }
- for (HistogramType histogramType : HistogramType.values()) {
- for (String histogramAttribute : histogramAttributes) {
- attributes.add(new MBeanAttributeInfo(
- histogramType.name() + "_" + histogramAttribute.toUpperCase(),
- "long", "RocksDBStat: " + histogramType.name(), true, false,
- false));
- }
- }
-
- return new MBeanInfo("", "RocksDBStat",
- attributes.toArray(new MBeanAttributeInfo[0]), null, null, null);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
deleted file mode 100644
index 4466337..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.utils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
deleted file mode 100644
index 5e3e0c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockClientServerProtocol.proto
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-option java_package = "org.apache.hadoop.cblock.protocol.proto";
-option java_outer_classname = "CBlockClientServerProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.cblock;
-
-import "Ozone.proto";
-import "CBlockServiceProtocol.proto";
-/**
-* This message is sent from CBlock client side to CBlock server to
-* mount a volume specified by owner name and volume name.
-*
-* Right now, this is the only communication between client and server.
-* After the volume is mounted, CBlock client will talk to containers
-* by itself, nothing to do with CBlock server.
-*/
-message MountVolumeRequestProto {
- required string userName = 1;
- required string volumeName = 2;
-}
-
-/**
-* This message is sent from CBlock server to CBlock client as response
-* of mount a volume. It checks the whether the volume is valid to access
-* at all.(e.g. volume exist)
-*
-* And include enough information (volume size, block size, list of
-* containers for this volume) for client side to perform read/write on
-* the volume.
-*/
-message MountVolumeResponseProto {
- required bool isValid = 1;
- optional string userName = 2;
- optional string volumeName = 3;
- optional uint64 volumeSize = 4;
- optional uint32 blockSize = 5;
- repeated ContainerIDProto allContainerIDs = 6;
-}
-
-/**
-* This message include ID of container which can be used to locate the
-* container. Since the order of containers needs to be maintained, also
-* includes a index field to verify the correctness of the order.
-*/
-message ContainerIDProto {
- required string containerID = 1;
- required uint64 index = 2;
- // making pipeline optional to be compatible with exisiting tests
- optional hadoop.hdfs.ozone.Pipeline pipeline = 3;
-}
-
-
-message ListVolumesRequestProto {
-
-}
-
-message ListVolumesResponseProto {
- repeated VolumeInfoProto volumeEntry = 1;
-}
-
-
-service CBlockClientServerProtocolService {
- /**
- * mount the volume.
- */
- rpc mountVolume(MountVolumeRequestProto) returns (MountVolumeResponseProto);
-
- rpc listVolumes(ListVolumesRequestProto) returns(ListVolumesResponseProto);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
deleted file mode 100644
index 36e4b59..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/CBlockServiceProtocol.proto
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.cblock.protocol.proto";
-option java_outer_classname = "CBlockServiceProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.cblock;
-
-/**
-* This message is sent to CBlock server to create a volume. Creating
-* volume requries four parameters: owner of the volume, name of the volume
-* size of volume and block size of the volume.
-*/
-message CreateVolumeRequestProto {
- required string userName = 1;
- required string volumeName = 2;
- required uint64 volumeSize = 3;
- optional uint32 blockSize = 4 [default = 4096];
-}
-
-/**
-* Empty response message.
-*/
-message CreateVolumeResponseProto {
-
-}
-
-/**
-* This message is sent to CBlock server to delete a volume. The volume
-* is specified by owner name and volume name. If force is set to
-* false, volume will be deleted only if it is empty. Otherwise delete it
-* regardless.
-*/
-message DeleteVolumeRequestProto {
- required string userName = 1;
- required string volumeName = 2;
- optional bool force = 3;
-}
-
-/**
-* Empty response message.
-*/
-message DeleteVolumeResponseProto {
-
-}
-
-/**
-* This message is sent to CBlock server to request info of a volume. The
-* volume is specified by owner name and volume name.
-*/
-message InfoVolumeRequestProto {
- required string userName = 1;
- required string volumeName = 2;
-}
-
-/**
-* This message describes the information of a volume.
-* Currently, the info includes the volume creation parameters and a number
-* as the usage of the volume, in terms of number of bytes.
-*/
-message VolumeInfoProto {
- required string userName = 1;
- required string volumeName = 2;
- required uint64 volumeSize = 3;
- required uint64 blockSize = 4;
- optional uint64 usage = 5;
- // TODO : potentially volume ACL
-}
-
-/**
-* This message is sent from CBlock server as response of info volume request.
-*/
-message InfoVolumeResponseProto {
- optional VolumeInfoProto volumeInfo = 1;
-}
-
-/**
-* This message is sent to CBlock server to list all available volume.
-*/
-message ListVolumeRequestProto {
- optional string userName = 1;
-}
-
-/**
-* This message is sent from CBlock server as response of volume listing.
-*/
-message ListVolumeResponseProto {
- repeated VolumeInfoProto volumeEntry = 1;
-}
-
-service CBlockServiceProtocolService {
- /**
- * Create a volume.
- */
- rpc createVolume(CreateVolumeRequestProto) returns(CreateVolumeResponseProto);
-
- /**
- * Delete a volume.
- */
- rpc deleteVolume(DeleteVolumeRequestProto) returns(DeleteVolumeResponseProto);
-
- /**
- * Get info of a volume.
- */
- rpc infoVolume(InfoVolumeRequestProto) returns(InfoVolumeResponseProto);
-
- /**
- * List all available volumes.
- */
- rpc listVolume(ListVolumeRequestProto) returns(ListVolumeResponseProto);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
index 35906c6..0af3c4f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/HdfsServerConstants.java
@@ -189,10 +189,8 @@ public interface HdfsServerConstants {
return NamenodeRole.NAMENODE;
}
}
-
+
public void setClusterId(String cid) {
- Preconditions.checkState(this == UPGRADE || this == UPGRADEONLY
- || this == FORMAT);
clusterId = cid;
}
@@ -217,7 +215,6 @@ public interface HdfsServerConstants {
}
public void setForce(int force) {
- Preconditions.checkState(this == RECOVER);
this.force = force;
}
@@ -230,7 +227,6 @@ public interface HdfsServerConstants {
}
public void setForceFormat(boolean force) {
- Preconditions.checkState(this == FORMAT);
isForceFormat = force;
}
@@ -239,7 +235,6 @@ public interface HdfsServerConstants {
}
public void setInteractiveFormat(boolean interactive) {
- Preconditions.checkState(this == FORMAT);
isInteractiveFormat = interactive;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1f51935..b757fd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -48,6 +48,7 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_P
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_METRICS_LOGGER_PERIOD_SECONDS_KEY;
import static org.apache.hadoop.util.ExitUtil.terminate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.protocol.proto.ReconfigurationProtocolProtos.ReconfigurationProtocolService;
@@ -109,10 +110,8 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.ozone.container.common.statemachine.DatanodeStateMachine;
import org.apache.hadoop.hdfs.server.datanode.checker.DatasetVolumeChecker;
import org.apache.hadoop.hdfs.server.datanode.checker.StorageLocationChecker;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
import org.apache.hadoop.util.AutoCloseableLock;
import org.apache.hadoop.hdfs.client.BlockReportOptions;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -191,7 +190,6 @@ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.DNS;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.unix.DomainSocket;
-import org.apache.hadoop.conf.OzoneConfiguration;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.SaslPropertiesResolver;
import org.apache.hadoop.security.SecurityUtil;
@@ -373,7 +371,6 @@ public class DataNode extends ReconfigurableBase
private final String confVersion;
private final long maxNumberOfBlocksToLog;
private final boolean pipelineSupportECN;
- private final boolean ozoneEnabled;
private final List<String> usersWithLocalPathAccess;
private final boolean connectToDnViaHostname;
@@ -402,8 +399,6 @@ public class DataNode extends ReconfigurableBase
private final SocketFactory socketFactory;
- private DatanodeStateMachine datanodeStateMachine;
-
private static Tracer createTracer(Configuration conf) {
return new Tracer.Builder("DataNode").
conf(TraceUtils.wrapHadoopConf(DATANODE_HTRACE_PREFIX , conf)).
@@ -414,8 +409,6 @@ public class DataNode extends ReconfigurableBase
private ScheduledThreadPoolExecutor metricsLoggerTimer;
- private ObjectStoreHandler objectStoreHandler = null;
-
/**
* Creates a dummy DataNode for testing purpose.
*/
@@ -434,7 +427,6 @@ public class DataNode extends ReconfigurableBase
this.connectToDnViaHostname = false;
this.blockScanner = new BlockScanner(this, this.getConf());
this.pipelineSupportECN = false;
- this.ozoneEnabled = false;
this.socketFactory = NetUtils.getDefaultSocketFactory(conf);
this.dnConf = new DNConf(this);
initOOBTimeout();
@@ -474,8 +466,6 @@ public class DataNode extends ReconfigurableBase
DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED,
DFSConfigKeys.DFS_PIPELINE_ECN_ENABLED_DEFAULT);
- this.ozoneEnabled = DFSUtil.isOzoneEnabled(conf);
-
confVersion = "core-" +
conf.get("hadoop.common.configuration.version", "UNSPECIFIED") +
",hdfs-" +
@@ -531,7 +521,7 @@ public class DataNode extends ReconfigurableBase
@Override // ReconfigurableBase
protected Configuration getNewConf() {
- return new OzoneConfiguration();
+ return new HdfsConfiguration();
}
/**
@@ -961,8 +951,8 @@ public class DataNode extends ReconfigurableBase
// the DN is started by JSVC, pass it along.
ServerSocketChannel httpServerChannel = secureResources != null ?
secureResources.getHttpServerChannel() : null;
- this.httpServer = new DatanodeHttpServer(getConf(), this, httpServerChannel,
- this.objectStoreHandler);
+
+ httpServer = new DatanodeHttpServer(getConf(), this, httpServerChannel);
httpServer.start();
if (httpServer.getHttpAddress() != null) {
infoPort = httpServer.getHttpAddress().getPort();
@@ -1411,9 +1401,7 @@ public class DataNode extends ReconfigurableBase
// global DN settings
registerMXBean();
-
initDataXceiver();
- initObjectStoreHandler();
startInfoServer();
pauseMonitor = new JvmPauseMonitor();
pauseMonitor.init(getConf());
@@ -1454,19 +1442,6 @@ public class DataNode extends ReconfigurableBase
}
/**
- * Initializes the object store handler. This must be called before
- * initialization of the HTTP server.
- *
- * @throws IOException if there is an I/O error
- */
- private void initObjectStoreHandler() throws IOException {
- if (this.ozoneEnabled) {
- this.objectStoreHandler = new ObjectStoreHandler(getConf());
- LOG.info("ozone is enabled.");
- }
- }
-
- /**
* Checks if the DataNode has a secure configuration if security is enabled.
* There are 2 possible configurations that are considered secure:
* 1. The server has bound to privileged ports for RPC and HTTP via
@@ -1578,6 +1553,11 @@ public class DataNode extends ReconfigurableBase
streamingAddr.getAddress().getHostAddress(), hostName,
storage.getDatanodeUuid(), getXferPort(), getInfoPort(),
infoSecurePort, getIpcPort());
+ for (ServicePlugin plugin : plugins) {
+ if (plugin instanceof DataNodeServicePlugin) {
+ ((DataNodeServicePlugin) plugin).onDatanodeIdCreation(dnId);
+ }
+ }
return new DatanodeRegistration(dnId, storageInfo,
new ExportedBlockKeys(), VersionInfo.getVersion());
}
@@ -1598,31 +1578,15 @@ public class DataNode extends ReconfigurableBase
+ ". Expecting " + storage.getDatanodeUuid());
}
- if (isOzoneEnabled()) {
- if (datanodeStateMachine == null) {
- datanodeStateMachine = new DatanodeStateMachine(
- getDatanodeId(),
- getConf());
- datanodeStateMachine.startDaemon();
+ for (ServicePlugin plugin : plugins) {
+ if (plugin instanceof DataNodeServicePlugin) {
+ ((DataNodeServicePlugin) plugin)
+ .onDatanodeSuccessfulNamenodeRegisration(bpRegistration);
}
}
registerBlockPoolWithSecretManager(bpRegistration, blockPoolId);
}
-
- @VisibleForTesting
- public OzoneContainer getOzoneContainerManager() {
- return this.datanodeStateMachine.getContainer();
- }
-
- @VisibleForTesting
- public DatanodeStateMachine.DatanodeStates getOzoneStateMachineState() {
- if (this.datanodeStateMachine != null) {
- return this.datanodeStateMachine.getContext().getState();
- }
- // if the state machine doesn't exist then DN initialization is in progress
- return DatanodeStateMachine.DatanodeStates.INIT;
- }
-
+
/**
* After the block pool has contacted the NN, registers that block pool
* with the secret manager, updating it with the secrets provided by the NN.
@@ -1729,11 +1693,11 @@ public class DataNode extends ReconfigurableBase
BPOfferService getBPOfferService(String bpid){
return blockPoolManager.get(bpid);
}
-
+
int getBpOsCount() {
return blockPoolManager.getAllNamenodeThreads().size();
}
-
+
/**
* Initializes the {@link #data}. The initialization is done only once, when
* handshake with the the first namenode is completed.
@@ -2021,7 +1985,7 @@ public class DataNode extends ReconfigurableBase
}
}
}
-
+
List<BPOfferService> bposArray = (this.blockPoolManager == null)
? new ArrayList<BPOfferService>()
: this.blockPoolManager.getAllNamenodeThreads();
@@ -2054,6 +2018,7 @@ public class DataNode extends ReconfigurableBase
// Terminate directory scanner and block scanner
shutdownPeriodicScanners();
+ shutdownDiskBalancer();
// Stop the web server
if (httpServer != null) {
@@ -2064,17 +2029,6 @@ public class DataNode extends ReconfigurableBase
}
}
- // Stop the object store handler
- if (isOzoneEnabled()) {
- if (this.objectStoreHandler != null) {
- this.objectStoreHandler.close();
- if(datanodeStateMachine != null &&
- !datanodeStateMachine.isDaemonStopped()) {
- datanodeStateMachine.stopDaemon();
- }
- }
- }
-
volumeChecker.shutdownAndWait(1, TimeUnit.SECONDS);
if (storageLocationChecker != null) {
@@ -2088,7 +2042,7 @@ public class DataNode extends ReconfigurableBase
// shouldRun is set to false here to prevent certain threads from exiting
// before the restart prep is done.
this.shouldRun = false;
-
+
// wait reconfiguration thread, if any, to exit
shutdownReconfigurationTask();
@@ -2098,8 +2052,9 @@ public class DataNode extends ReconfigurableBase
while (true) {
// When shutting down for restart, wait 2.5 seconds before forcing
// termination of receiver threads.
- if (!this.shutdownForUpgrade || (this.shutdownForUpgrade && (
- Time.monotonicNow() - timeNotified > 1000))) {
+ if (!this.shutdownForUpgrade ||
+ (this.shutdownForUpgrade && (Time.monotonicNow() - timeNotified
+ > 1000))) {
this.threadGroup.interrupt();
break;
}
@@ -2110,8 +2065,7 @@ public class DataNode extends ReconfigurableBase
}
try {
Thread.sleep(sleepMs);
- } catch (InterruptedException e) {
- }
+ } catch (InterruptedException e) {}
sleepMs = sleepMs * 3 / 2; // exponential backoff
if (sleepMs > 200) {
sleepMs = 200;
@@ -2137,9 +2091,9 @@ public class DataNode extends ReconfigurableBase
metrics.setDataNodeActiveXceiversCount(0);
}
- // IPC server needs to be shutdown late in the process, otherwise
- // shutdown command response won't get sent.
- if (ipcServer != null) {
+ // IPC server needs to be shutdown late in the process, otherwise
+ // shutdown command response won't get sent.
+ if (ipcServer != null) {
ipcServer.stop();
}
@@ -2154,7 +2108,7 @@ public class DataNode extends ReconfigurableBase
LOG.warn("Received exception in BlockPoolManager#shutDownAll", ie);
}
}
-
+
if (storage != null) {
try {
this.storage.unlockAll();
@@ -2175,11 +2129,9 @@ public class DataNode extends ReconfigurableBase
MBeans.unregister(dataNodeInfoBeanName);
dataNodeInfoBeanName = null;
}
- if (shortCircuitRegistry != null) {
- shortCircuitRegistry.shutdown();
- }
+ if (shortCircuitRegistry != null) shortCircuitRegistry.shutdown();
LOG.info("Shutdown complete.");
- synchronized (this) {
+ synchronized(this) {
// it is already false, but setting it again to avoid a findbug warning.
this.shouldRun = false;
// Notify the main thread.
@@ -2717,9 +2669,8 @@ public class DataNode extends ReconfigurableBase
*/
public static DataNode instantiateDataNode(String args [], Configuration conf,
SecureResources resources) throws IOException {
- if (conf == null) {
- conf = new OzoneConfiguration();
- }
+ if (conf == null)
+ conf = new HdfsConfiguration();
if (args != null) {
// parse generic hadoop options
@@ -3252,12 +3203,6 @@ public class DataNode extends ReconfigurableBase
} catch (InterruptedException ie) { }
}
shutdown();
-
- if (isOzoneEnabled()) {
- if(datanodeStateMachine != null) {
- datanodeStateMachine.stopDaemon();
- }
- }
}
};
@@ -3552,13 +3497,10 @@ public class DataNode extends ReconfigurableBase
return metricsLoggerTimer;
}
- public boolean isOzoneEnabled() {
- return ozoneEnabled;
- }
-
public Tracer getTracer() {
return tracer;
}
+
/**
* Allows submission of a disk balancer Job.
* @param planID - Hash value of the plan.
@@ -3676,4 +3618,14 @@ public class DataNode extends ReconfigurableBase
}
return volumeInfoList;
}
+
+ @Private
+ public SecureResources getSecureResources() {
+ return secureResources;
+ }
+
+ @Private
+ public Collection<ServicePlugin> getPlugins() {
+ return Collections.unmodifiableList(plugins);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
new file mode 100644
index 0000000..08aae8b
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNodeServicePlugin.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.datanode;
+
+import org.apache.hadoop.hdfs.protocol.DatanodeID;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.util.ServicePlugin;
+
+/**
+ * Datanode specific service plugin with additional hooks.
+ */
+public interface DataNodeServicePlugin extends ServicePlugin{
+
+ /**
+ * Extension point to modify the datanode id.
+ *
+ * @param dataNodeId
+ */
+ default void onDatanodeIdCreation(DatanodeID dataNodeId) {
+ //NOOP
+ }
+
+ /**
+ * Extension point to modify the datanode id.
+ *
+ * @param dataNodeId
+ */
+ default void onDatanodeSuccessfulNamenodeRegisration(
+ DatanodeRegistration dataNodeId) {
+ //NOOP
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
deleted file mode 100644
index a302462..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ObjectStoreHandler.java
+++ /dev/null
@@ -1,189 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.datanode;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.*;
-import static com.sun.jersey.api.core.ResourceConfig.PROPERTY_CONTAINER_REQUEST_FILTERS;
-import static com.sun.jersey.api.core.ResourceConfig.FEATURE_TRACE;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.HashMap;
-import java.util.Map;
-
-import com.sun.jersey.api.container.ContainerFactory;
-import com.sun.jersey.api.core.ApplicationAdapter;
-
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.ksm.protocolPB
- .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.protocolPB
- .ScmBlockLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.ObjectStoreApplication;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
-import org.apache.hadoop.ozone.web.storage.DistributedStorageHandler;
-import org.apache.hadoop.ozone.web.localstorage.LocalStorageHandler;
-import org.apache.hadoop.security.UserGroupInformation;
-
-/**
- * Implements object store handling within the DataNode process. This class is
- * responsible for initializing and maintaining the RPC clients and servers and
- * the web application required for the object store implementation.
- */
-public final class ObjectStoreHandler implements Closeable {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ObjectStoreJerseyContainer.class);
-
- private final ObjectStoreJerseyContainer objectStoreJerseyContainer;
- private final KeySpaceManagerProtocolClientSideTranslatorPB
- keySpaceManagerClient;
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final ScmBlockLocationProtocolClientSideTranslatorPB
- scmBlockLocationClient;
- private final StorageHandler storageHandler;
-
- /**
- * Creates a new ObjectStoreHandler.
- *
- * @param conf configuration
- * @throws IOException if there is an I/O error
- */
- public ObjectStoreHandler(Configuration conf) throws IOException {
- String shType = conf.getTrimmed(OZONE_HANDLER_TYPE_KEY,
- OZONE_HANDLER_TYPE_DEFAULT);
- LOG.info("ObjectStoreHandler initializing with {}: {}",
- OZONE_HANDLER_TYPE_KEY, shType);
- boolean ozoneTrace = conf.getBoolean(OZONE_TRACE_ENABLED_KEY,
- OZONE_TRACE_ENABLED_DEFAULT);
-
- // Initialize Jersey container for object store web application.
- if (OzoneConsts.OZONE_HANDLER_DISTRIBUTED.equalsIgnoreCase(shType)) {
- RPC.setProtocolEngine(conf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- long scmVersion =
- RPC.getProtocolVersion(StorageContainerLocationProtocolPB.class);
-
- InetSocketAddress scmAddress =
- OzoneClientUtils.getScmAddressForClients(conf);
- this.storageContainerLocationClient =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, scmVersion,
- scmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- InetSocketAddress scmBlockAddress =
- OzoneClientUtils.getScmAddressForBlockClients(conf);
- this.scmBlockLocationClient =
- new ScmBlockLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(ScmBlockLocationProtocolPB.class, scmVersion,
- scmBlockAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- RPC.setProtocolEngine(conf, KeySpaceManagerProtocolPB.class,
- ProtobufRpcEngine.class);
- long ksmVersion =
- RPC.getProtocolVersion(KeySpaceManagerProtocolPB.class);
- InetSocketAddress ksmAddress = OzoneClientUtils.getKsmAddress(conf);
- this.keySpaceManagerClient =
- new KeySpaceManagerProtocolClientSideTranslatorPB(
- RPC.getProxy(KeySpaceManagerProtocolPB.class, ksmVersion,
- ksmAddress, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf),
- Client.getRpcTimeout(conf)));
-
- storageHandler = new DistributedStorageHandler(
- new OzoneConfiguration(conf),
- this.storageContainerLocationClient,
- this.keySpaceManagerClient);
- } else {
- if (OzoneConsts.OZONE_HANDLER_LOCAL.equalsIgnoreCase(shType)) {
- storageHandler = new LocalStorageHandler(conf);
- this.storageContainerLocationClient = null;
- this.scmBlockLocationClient = null;
- this.keySpaceManagerClient = null;
- } else {
- throw new IllegalArgumentException(
- String.format("Unrecognized value for %s: %s,"
- + " Allowed values are %s,%s",
- OZONE_HANDLER_TYPE_KEY, shType,
- OzoneConsts.OZONE_HANDLER_DISTRIBUTED,
- OzoneConsts.OZONE_HANDLER_LOCAL));
- }
- }
- ApplicationAdapter aa =
- new ApplicationAdapter(new ObjectStoreApplication());
- Map<String, Object> settingsMap = new HashMap<>();
- settingsMap.put(PROPERTY_CONTAINER_REQUEST_FILTERS,
- ServiceFilter.class.getCanonicalName());
- settingsMap.put(FEATURE_TRACE, ozoneTrace);
- aa.setPropertiesAndFeatures(settingsMap);
- this.objectStoreJerseyContainer = ContainerFactory.createContainer(
- ObjectStoreJerseyContainer.class, aa);
- this.objectStoreJerseyContainer.setStorageHandler(storageHandler);
- }
-
- /**
- * Returns the initialized web application container.
- *
- * @return initialized web application container
- */
- public ObjectStoreJerseyContainer getObjectStoreJerseyContainer() {
- return this.objectStoreJerseyContainer;
- }
-
- /**
- * Returns the storage handler.
- *
- * @return returns the storage handler
- */
- public StorageHandler getStorageHandler() {
- return this.storageHandler;
- }
-
- @Override
- public void close() {
- LOG.info("Closing ObjectStoreHandler.");
- storageHandler.close();
- IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
- IOUtils.cleanupWithLogger(LOG, scmBlockLocationClient);
- IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 1852192..c141293 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1745,7 +1745,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
}
assert newReplicaInfo.getState() == ReplicaState.FINALIZED
: "Replica should be finalized";
-
+
volumeMap.add(bpid, newReplicaInfo);
return newReplicaInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
index dbc03c2..2e46b28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/DatanodeHttpServer.java
@@ -51,11 +51,9 @@ import org.apache.hadoop.hdfs.server.common.JspHelper;
import org.apache.hadoop.hdfs.server.datanode.BlockScanner;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.DataNodeUGIProvider;
-import org.apache.hadoop.hdfs.server.datanode.ObjectStoreHandler;
import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.http.RestCsrfPreventionFilter;
import org.apache.hadoop.security.ssl.SSLFactory;
@@ -93,19 +91,11 @@ public class DatanodeHttpServer implements Closeable {
public DatanodeHttpServer(final Configuration conf,
final DataNode datanode,
- final ServerSocketChannel externalHttpChannel,
- final ObjectStoreHandler objectStoreHandler)
+ final ServerSocketChannel externalHttpChannel)
throws IOException {
this.restCsrfPreventionFilter = createRestCsrfPreventionFilter(conf);
this.conf = conf;
- final ObjectStoreJerseyContainer finalContainer;
- if (objectStoreHandler != null) {
- finalContainer = objectStoreHandler.getObjectStoreJerseyContainer();
- } else {
- finalContainer = null;
- }
-
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS_KEY, 10);
int proxyPort =
@@ -160,8 +150,7 @@ public class DatanodeHttpServer implements Closeable {
}
p.addLast(
new ChunkedWriteHandler(),
- new URLDispatcher(jettyAddr, conf, confForCreate,
- finalContainer));
+ new URLDispatcher(jettyAddr, conf, confForCreate));
}
});
@@ -218,8 +207,7 @@ public class DatanodeHttpServer implements Closeable {
}
p.addLast(
new ChunkedWriteHandler(),
- new URLDispatcher(jettyAddr, conf, confForCreate,
- finalContainer));
+ new URLDispatcher(jettyAddr, conf, confForCreate));
}
});
} else {
@@ -386,4 +374,3 @@ public class DatanodeHttpServer implements Closeable {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
index 4958bb5..e275fb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/RestCsrfPreventionFilterHandler.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.security.http.RestCsrfPreventionFilter.HttpInteraction;
* handler drops the request and immediately sends an HTTP 400 response.
*/
@InterfaceAudience.Private
-final class RestCsrfPreventionFilterHandler
+public final class RestCsrfPreventionFilterHandler
extends SimpleChannelInboundHandler<HttpRequest> {
private static final Log LOG = DatanodeHttpServer.LOG;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
index dd958d1..8ec5bf6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/URLDispatcher.java
@@ -17,106 +17,42 @@
*/
package org.apache.hadoop.hdfs.server.datanode.web;
-import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
-
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelPipeline;
import io.netty.channel.SimpleChannelInboundHandler;
import io.netty.handler.codec.http.HttpRequest;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.netty.ObjectStoreJerseyContainer;
-import org.apache.hadoop.ozone.web.netty.RequestDispatchObjectStoreChannelHandler;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import java.io.IOException;
import java.net.InetSocketAddress;
+import static org.apache.hadoop.hdfs.server.datanode.web.webhdfs.WebHdfsHandler.WEBHDFS_PREFIX;
+
class URLDispatcher extends SimpleChannelInboundHandler<HttpRequest> {
- protected static final Logger LOG =
- LoggerFactory.getLogger(URLDispatcher.class);
private final InetSocketAddress proxyHost;
private final Configuration conf;
private final Configuration confForCreate;
- private final ObjectStoreJerseyContainer objectStoreJerseyContainer;
URLDispatcher(InetSocketAddress proxyHost, Configuration conf,
- Configuration confForCreate,
- ObjectStoreJerseyContainer objectStoreJerseyContainer)
- throws IOException {
+ Configuration confForCreate) {
this.proxyHost = proxyHost;
this.conf = conf;
this.confForCreate = confForCreate;
- this.objectStoreJerseyContainer = objectStoreJerseyContainer;
}
@Override
protected void channelRead0(ChannelHandlerContext ctx, HttpRequest req)
throws Exception {
+ String uri = req.getUri();
ChannelPipeline p = ctx.pipeline();
- if (isWebHdfsRequest(req)) {
+ if (uri.startsWith(WEBHDFS_PREFIX)) {
WebHdfsHandler h = new WebHdfsHandler(conf, confForCreate);
p.replace(this, WebHdfsHandler.class.getSimpleName(), h);
h.channelRead0(ctx, req);
- } else if (isObjectStoreRequest(req)) {
- RequestDispatchObjectStoreChannelHandler h =
- new RequestDispatchObjectStoreChannelHandler(
- this.objectStoreJerseyContainer);
- p.replace(this,
- RequestDispatchObjectStoreChannelHandler.class.getSimpleName(), h);
- h.channelRead0(ctx, req);
- } else if (!isObjectStoreRequestHeaders(req)){
+ } else {
SimpleHttpProxyHandler h = new SimpleHttpProxyHandler(proxyHost);
p.replace(this, SimpleHttpProxyHandler.class.getSimpleName(), h);
h.channelRead0(ctx, req);
}
}
-
-
- /*
- * Returns true if the request has ozone headers
- *
- * @param req HTTP request
- * @return true if request has ozone header, else false
- */
-
- private boolean isObjectStoreRequestHeaders(HttpRequest req) {
- for (String version : req.headers().getAll(Header.OZONE_VERSION_HEADER)) {
- if (version != null) {
- LOG.debug("ozone : dispatching call to Ozone, when security is not " +
- "enabled");
- return true;
- }
- }
- return false;
- }
-
-
- /*
- * Returns true if the request is to be handled by the object store.
- *
- * @param req HTTP request
- * @return true if the request is to be handled by the object store
- */
- private boolean isObjectStoreRequest(HttpRequest req) {
- if (this.objectStoreJerseyContainer == null) {
- LOG.debug("ozone : ozone is disabled or when security is enabled, ozone" +
- " is not supported");
- return false;
- }
- return isObjectStoreRequestHeaders(req);
- }
-
- /**
- * Returns true if the request is to be handled by WebHDFS.
- *
- * @param req HTTP request
- * @return true if the request is to be handled by WebHDFS
- */
- private boolean isWebHdfsRequest(HttpRequest req) {
- return req.getUri().startsWith(WEBHDFS_PREFIX);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
index 2debf2e..10650da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/NamespaceInfo.java
@@ -34,9 +34,9 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
- * NamespaceInfo is returned by the name-node in reply
+ * NamespaceInfo is returned by the name-node in reply
* to a data-node handshake.
- *
+ *
*/
@InterfaceAudience.Private
@InterfaceStability.Evolving
@@ -110,7 +110,7 @@ public class NamespaceInfo extends StorageInfo {
this.capabilities = capabilities;
}
- public NamespaceInfo(int nsID, String clusterID, String bpID,
+ public NamespaceInfo(int nsID, String clusterID, String bpID,
long cT) {
this(nsID, clusterID, bpID, cT, Storage.getBuildVersion(),
VersionInfo.getVersion());
@@ -122,7 +122,7 @@ public class NamespaceInfo extends StorageInfo {
VersionInfo.getVersion());
this.state = st;
}
-
+
public long getCapabilities() {
return capabilities;
}
@@ -151,7 +151,7 @@ public class NamespaceInfo extends StorageInfo {
public String getBlockPoolID() {
return blockPoolID;
}
-
+
public String getSoftwareVersion() {
return softwareVersion;
}
@@ -194,4 +194,4 @@ public class NamespaceInfo extends StorageInfo {
"BPID=" + storage.getBlockPoolID() + ".");
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
index a1ea3fb..aaa1038 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/GetConf.java
@@ -26,7 +26,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Collection;
import java.util.Set;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -36,8 +35,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.DFSUtil.ConfiguredNNAddress;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -81,10 +78,6 @@ public class GetConf extends Configured implements Tool {
"gets the exclude file path that defines the datanodes " +
"that need to decommissioned."),
NNRPCADDRESSES("-nnRpcAddresses", "gets the namenode rpc addresses"),
- KEYSPACEMANAGER("-keyspacemanagers",
- "gets list of ozone key space manager nodes in the cluster"),
- STORAGECONTAINERMANAGER("-storagecontainermanagers",
- "gets list of ozone storage container manager nodes in the cluster"),
CONFKEY("-confKey [key]", "gets a specific key from the configuration");
private static final Map<String, CommandHandler> map;
@@ -104,10 +97,6 @@ public class GetConf extends Configured implements Tool {
new CommandHandler(DFSConfigKeys.DFS_HOSTS_EXCLUDE));
map.put(StringUtils.toLowerCase(NNRPCADDRESSES.getName()),
new NNRpcAddressesCommandHandler());
- map.put(StringUtils.toLowerCase(KEYSPACEMANAGER.getName()),
- new KeySpaceManagersCommandHandler());
- map.put(StringUtils.toLowerCase(STORAGECONTAINERMANAGER.getName()),
- new StorageContainerManagersCommandHandler());
map.put(StringUtils.toLowerCase(CONFKEY.getName()),
new PrintConfKeyCommandHandler());
}
@@ -235,36 +224,9 @@ public class GetConf extends Configured implements Tool {
* Handler for {@link Command#SECONDARY}
*/
static class SecondaryNameNodesCommandHandler extends CommandHandler {
- @Override public int doWorkInternal(GetConf tool, String[] args)
- throws IOException {
- tool.printMap(DFSUtil.getSecondaryNameNodeAddresses(tool.getConf()));
- return 0;
- }
- }
-
- /**
- * Handler for {@link Command#STORAGECONTAINERMANAGER}.
- */
- static class StorageContainerManagersCommandHandler extends CommandHandler {
@Override
- public int doWorkInternal(GetConf tool, String[] args) throws IOException {
- Collection<InetSocketAddress> addresses =
- OzoneClientUtils.getSCMAddresses(tool.getConf());
- for (InetSocketAddress addr : addresses) {
- tool.printOut(addr.getHostName());
- }
- return 0;
- }
- }
-
- /**
- * Handler for {@link Command#KEYSPACEMANAGER}.
- */
- static class KeySpaceManagersCommandHandler extends CommandHandler {
- @Override
- public int doWorkInternal(GetConf tool, String[] args) throws IOException {
- tool.printOut(OzoneClientUtils.getKsmAddress(tool.getConf())
- .getHostName());
+ public int doWorkInternal(GetConf tool, String []args) throws IOException {
+ tool.printMap(DFSUtil.getSecondaryNameNodeAddresses(tool.getConf()));
return 0;
}
}
@@ -395,11 +357,8 @@ public class GetConf extends Configured implements Tool {
if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
System.exit(0);
}
-
- Configuration conf = new Configuration();
- conf.addResource(new HdfsConfiguration());
- conf.addResource(new OzoneConfiguration());
- int res = ToolRunner.run(new GetConf(conf), args);
+
+ int res = ToolRunner.run(new GetConf(new HdfsConfiguration()), args);
System.exit(res);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
deleted file mode 100644
index c3f9234..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/InconsistentStorageStateException.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import java.io.File;
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * The exception is thrown when file system state is inconsistent
- * and is not recoverable.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class InconsistentStorageStateException extends IOException {
- private static final long serialVersionUID = 1L;
-
- public InconsistentStorageStateException(String descr) {
- super(descr);
- }
-
- public InconsistentStorageStateException(File dir, String descr) {
- super("Directory " + getFilePath(dir) + " is in an inconsistent state: "
- + descr);
- }
-
- private static String getFilePath(File dir) {
- try {
- return dir.getCanonicalPath();
- } catch (IOException e) {
- }
- return dir.getPath();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
deleted file mode 100644
index cfcddb4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/Storage.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.DirectoryStream;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.util.Properties;
-
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.util.Time;
-
-/**
- * Storage information file. This Class defines the methods to check
- * the consistency of the storage dir and the version file.
- * <p>
- * Local storage information is stored in a separate file VERSION.
- * It contains type of the node,
- * the storage layout version, the SCM id, and
- * the KSM/SCM state creation time.
- *
- */
-@InterfaceAudience.Private
-public abstract class Storage {
- private static final Logger LOG = LoggerFactory.getLogger(Storage.class);
-
- protected static final String STORAGE_DIR_CURRENT = "current";
- protected static final String STORAGE_FILE_VERSION = "VERSION";
-
- private final NodeType nodeType;
- private final File root;
- private final File storageDir;
-
- private StorageState state;
- private StorageInfo storageInfo;
-
-
- /**
- * Determines the state of the Version file.
- */
- public enum StorageState {
- NON_EXISTENT, NOT_INITIALIZED, INITIALIZED
- }
-
- public Storage(NodeType type, File root, String sdName)
- throws IOException {
- this.nodeType = type;
- this.root = root;
- this.storageDir = new File(root, sdName);
- this.state = getStorageState();
- if (state == StorageState.INITIALIZED) {
- this.storageInfo = new StorageInfo(type, getVersionFile());
- } else {
- this.storageInfo = new StorageInfo(
- nodeType, StorageInfo.newClusterID(), Time.now());
- setNodeProperties();
- }
- }
-
- /**
- * Gets the path of the Storage dir.
- * @return Stoarge dir path
- */
- public String getStorageDir() {
- return storageDir.getAbsoluteFile().toString();
- }
-
- /**
- * Gets the state of the version file.
- * @return the state of the Version file
- */
- public StorageState getState() {
- return state;
- }
-
- public NodeType getNodeType() {
- return storageInfo.getNodeType();
- }
-
- public String getClusterID() {
- return storageInfo.getClusterID();
- }
-
- public long getCreationTime() {
- return storageInfo.getCreationTime();
- }
-
- public void setClusterId(String clusterId) throws IOException {
- if (state == StorageState.INITIALIZED) {
- throw new IOException(
- "Storage directory " + storageDir + " already initialized.");
- } else {
- storageInfo.setClusterId(clusterId);
- }
- }
-
- /**
- * Retreives the storageInfo instance to read/write the common
- * version file properties.
- * @return the instance of the storageInfo class
- */
- protected StorageInfo getStorageInfo() {
- return storageInfo;
- }
-
- abstract protected Properties getNodeProperties();
-
- /**
- * Sets the Node properties spaecific to KSM/SCM.
- */
- private void setNodeProperties() {
- Properties nodeProperties = getNodeProperties();
- if (nodeProperties != null) {
- for (String key : nodeProperties.stringPropertyNames()) {
- storageInfo.setProperty(key, nodeProperties.getProperty(key));
- }
- }
- }
-
- /**
- * Directory {@code current} contains latest files defining
- * the file system meta-data.
- *
- * @return the directory path
- */
- private File getCurrentDir() {
- return new File(storageDir, STORAGE_DIR_CURRENT);
- }
-
- /**
- * File {@code VERSION} contains the following fields:
- * <ol>
- * <li>node type</li>
- * <li>KSM/SCM state creation time</li>
- * <li>other fields specific for this node type</li>
- * </ol>
- * The version file is always written last during storage directory updates.
- * The existence of the version file indicates that all other files have
- * been successfully written in the storage directory, the storage is valid
- * and does not need to be recovered.
- *
- * @return the version file path
- */
- private File getVersionFile() {
- return new File(getCurrentDir(), STORAGE_FILE_VERSION);
- }
-
-
- /**
- * Check to see if current/ directory is empty. This method is used
- * before determining to format the directory.
- * @throws IOException if unable to list files under the directory.
- */
- private void checkEmptyCurrent() throws IOException {
- File currentDir = getCurrentDir();
- if (!currentDir.exists()) {
- // if current/ does not exist, it's safe to format it.
- return;
- }
- try (DirectoryStream<Path> dirStream = Files
- .newDirectoryStream(currentDir.toPath())) {
- if (dirStream.iterator().hasNext()) {
- throw new InconsistentStorageStateException(getCurrentDir(),
- "Can't initialize the storage directory because the current "
- + "it is not empty.");
- }
- }
- }
-
- /**
- * Check consistency of the storage directory.
- *
- * @return state {@link StorageState} of the storage directory
- * @throws IOException
- */
- private StorageState getStorageState() throws IOException {
- assert root != null : "root is null";
- String rootPath = root.getCanonicalPath();
- try { // check that storage exists
- if (!root.exists()) {
- // storage directory does not exist
- LOG.warn("Storage directory " + rootPath + " does not exist");
- return StorageState.NON_EXISTENT;
- }
- // or is inaccessible
- if (!root.isDirectory()) {
- LOG.warn(rootPath + "is not a directory");
- return StorageState.NON_EXISTENT;
- }
- if (!FileUtil.canWrite(root)) {
- LOG.warn("Cannot access storage directory " + rootPath);
- return StorageState.NON_EXISTENT;
- }
- } catch (SecurityException ex) {
- LOG.warn("Cannot access storage directory " + rootPath, ex);
- return StorageState.NON_EXISTENT;
- }
-
- // check whether current directory is valid
- File versionFile = getVersionFile();
- boolean hasCurrent = versionFile.exists();
-
- if (hasCurrent) {
- return StorageState.INITIALIZED;
- } else {
- checkEmptyCurrent();
- return StorageState.NOT_INITIALIZED;
- }
- }
-
- /**
- * Creates the Version file if not present,
- * otherwise returns with IOException.
- * @throws IOException
- */
- public void initialize() throws IOException {
- if (state == StorageState.INITIALIZED) {
- throw new IOException("Storage directory already initialized.");
- }
- if (!getCurrentDir().mkdirs()) {
- throw new IOException("Cannot create directory " + getCurrentDir());
- }
- storageInfo.writeTo(getVersionFile());
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
deleted file mode 100644
index 82ad955..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/StorageInfo.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
-
-import java.io.IOException;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.File;
-import java.io.RandomAccessFile;
-
-import java.util.Properties;
-import java.util.UUID;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-
-/**
- * Common class for storage information. This class defines the common
- * properties and functions to set them , write them into the version file
- * and read them from the version file.
- *
- */
-@InterfaceAudience.Private
-public class StorageInfo {
-
- private Properties properties = new Properties();
-
- /**
- * Property to hold node type.
- */
- private static final String NODE_TYPE = "nodeType";
- /**
- * Property to hold ID of the cluster.
- */
- private static final String CLUSTER_ID = "clusterID";
- /**
- * Property to hold creation time of the storage.
- */
- private static final String CREATION_TIME = "cTime";
-
- /**
- * Constructs StorageInfo instance.
- * @param type
- * Type of the node using the storage
- * @param cid
- * Cluster ID
- * @param cT
- * Cluster creation Time
-
- * @throws IOException
- */
- public StorageInfo(NodeType type, String cid, long cT)
- throws IOException {
- Preconditions.checkNotNull(type);
- Preconditions.checkNotNull(cid);
- Preconditions.checkNotNull(cT);
- properties.setProperty(NODE_TYPE, type.name());
- properties.setProperty(CLUSTER_ID, cid);
- properties.setProperty(CREATION_TIME, String.valueOf(cT));
- }
-
- public StorageInfo(NodeType type, File propertiesFile)
- throws IOException {
- this.properties = readFrom(propertiesFile);
- verifyNodeType(type);
- verifyClusterId();
- verifyCreationTime();
- }
-
- public NodeType getNodeType() {
- return NodeType.valueOf(properties.getProperty(NODE_TYPE));
- }
-
- public String getClusterID() {
- return properties.getProperty(CLUSTER_ID);
- }
-
- public Long getCreationTime() {
- String creationTime = properties.getProperty(CREATION_TIME);
- if(creationTime != null) {
- return Long.parseLong(creationTime);
- }
- return null;
- }
-
- public String getProperty(String key) {
- return properties.getProperty(key);
- }
-
- public void setProperty(String key, String value) {
- properties.setProperty(key, value);
- }
-
- public void setClusterId(String clusterId) {
- properties.setProperty(CLUSTER_ID, clusterId);
- }
-
- private void verifyNodeType(NodeType type)
- throws InconsistentStorageStateException {
- NodeType nodeType = getNodeType();
- Preconditions.checkNotNull(nodeType);
- if(type != nodeType) {
- throw new InconsistentStorageStateException("Expected NodeType: " + type +
- ", but found: " + nodeType);
- }
- }
-
- private void verifyClusterId()
- throws InconsistentStorageStateException {
- String clusterId = getClusterID();
- Preconditions.checkNotNull(clusterId);
- if(clusterId.isEmpty()) {
- throw new InconsistentStorageStateException("Cluster ID not found");
- }
- }
-
- private void verifyCreationTime() {
- Long creationTime = getCreationTime();
- Preconditions.checkNotNull(creationTime);
- }
-
-
- public void writeTo(File to)
- throws IOException {
- try (RandomAccessFile file = new RandomAccessFile(to, "rws");
- FileOutputStream out = new FileOutputStream(file.getFD())) {
- file.seek(0);
- /*
- * If server is interrupted before this line,
- * the version file will remain unchanged.
- */
- properties.store(out, null);
- /*
- * Now the new fields are flushed to the head of the file, but file
- * length can still be larger then required and therefore the file can
- * contain whole or corrupted fields from its old contents in the end.
- * If server is interrupted here and restarted later these extra fields
- * either should not effect server behavior or should be handled
- * by the server correctly.
- */
- file.setLength(out.getChannel().position());
- }
- }
-
- private Properties readFrom(File from) throws IOException {
- try (RandomAccessFile file = new RandomAccessFile(from, "rws");
- FileInputStream in = new FileInputStream(file.getFD())) {
- Properties props = new Properties();
- file.seek(0);
- props.load(in);
- return props;
- }
- }
-
- /**
- * Generate new clusterID.
- *
- * clusterID is a persistent attribute of the cluster.
- * It is generated when the cluster is created and remains the same
- * during the life cycle of the cluster. When a new SCM node is initialized,
- * if this is a new cluster, a new clusterID is generated and stored.
- * @return new clusterID
- */
- public static String newClusterID() {
- return "CID-" + UUID.randomUUID().toString();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
deleted file mode 100644
index 6517e58..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/common/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.common;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
deleted file mode 100644
index 690e3b5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkInfo.java
+++ /dev/null
@@ -1,185 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Java class that represents ChunkInfo ProtoBuf class. This helper class allows
- * us to convert to and from protobuf to normal java.
- */
-public class ChunkInfo {
- private final String chunkName;
- private final long offset;
- private final long len;
- private String checksum;
- private final Map<String, String> metadata;
-
-
- /**
- * Constructs a ChunkInfo.
- *
- * @param chunkName - File Name where chunk lives.
- * @param offset - offset where Chunk Starts.
- * @param len - Length of the Chunk.
- */
- public ChunkInfo(String chunkName, long offset, long len) {
- this.chunkName = chunkName;
- this.offset = offset;
- this.len = len;
- this.metadata = new TreeMap<>();
- }
-
- /**
- * Adds metadata.
- *
- * @param key - Key Name.
- * @param value - Value.
- * @throws IOException
- */
- public void addMetadata(String key, String value) throws IOException {
- synchronized (this.metadata) {
- if (this.metadata.containsKey(key)) {
- throw new IOException("This key already exists. Key " + key);
- }
- metadata.put(key, value);
- }
- }
-
- /**
- * Gets a Chunkinfo class from the protobuf definitions.
- *
- * @param info - Protobuf class
- * @return ChunkInfo
- * @throws IOException
- */
- public static ChunkInfo getFromProtoBuf(ContainerProtos.ChunkInfo info)
- throws IOException {
- Preconditions.checkNotNull(info);
-
- ChunkInfo chunkInfo = new ChunkInfo(info.getChunkName(), info.getOffset(),
- info.getLen());
-
- for (int x = 0; x < info.getMetadataCount(); x++) {
- chunkInfo.addMetadata(info.getMetadata(x).getKey(),
- info.getMetadata(x).getValue());
- }
-
-
- if (info.hasChecksum()) {
- chunkInfo.setChecksum(info.getChecksum());
- }
- return chunkInfo;
- }
-
- /**
- * Returns a ProtoBuf Message from ChunkInfo.
- *
- * @return Protocol Buffer Message
- */
- public ContainerProtos.ChunkInfo getProtoBufMessage() {
- ContainerProtos.ChunkInfo.Builder builder = ContainerProtos
- .ChunkInfo.newBuilder();
-
- builder.setChunkName(this.getChunkName());
- builder.setOffset(this.getOffset());
- builder.setLen(this.getLen());
- if (this.getChecksum() != null && !this.getChecksum().isEmpty()) {
- builder.setChecksum(this.getChecksum());
- }
-
- for (Map.Entry<String, String> entry : metadata.entrySet()) {
- OzoneProtos.KeyValue.Builder keyValBuilder =
- OzoneProtos.KeyValue.newBuilder();
- builder.addMetadata(keyValBuilder.setKey(entry.getKey())
- .setValue(entry.getValue()).build());
- }
-
- return builder.build();
- }
-
- /**
- * Returns the chunkName.
- *
- * @return - String
- */
- public String getChunkName() {
- return chunkName;
- }
-
- /**
- * Gets the start offset of the given chunk in physical file.
- *
- * @return - long
- */
- public long getOffset() {
- return offset;
- }
-
- /**
- * Returns the length of the Chunk.
- *
- * @return long
- */
- public long getLen() {
- return len;
- }
-
- /**
- * Returns the SHA256 value of this chunk.
- *
- * @return - Hash String
- */
- public String getChecksum() {
- return checksum;
- }
-
- /**
- * Sets the Hash value of this chunk.
- *
- * @param checksum - Hash String.
- */
- public void setChecksum(String checksum) {
- this.checksum = checksum;
- }
-
- /**
- * Returns Metadata associated with this Chunk.
- *
- * @return - Map of Key,values.
- */
- public Map<String, String> getMetadata() {
- return metadata;
- }
-
- @Override
- public String toString() {
- return "ChunkInfo{" +
- "chunkName='" + chunkName +
- ", offset=" + offset +
- ", len=" + len +
- '}';
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
deleted file mode 100644
index 4eb211b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ChunkUtils.java
+++ /dev/null
@@ -1,345 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.AsynchronousFileChannel;
-import java.nio.channels.FileLock;
-import java.nio.file.StandardOpenOption;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CHECKSUM_MISMATCH;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_NOT_FOUND;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.INVALID_WRITE_SIZE;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.IO_EXCEPTION;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.OVERWRITE_FLAG_REQUIRED;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNABLE_TO_FIND_CHUNK;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNABLE_TO_FIND_DATA_DIR;
-
-/**
- * Set of utility functions used by the chunk Manager.
- */
-public final class ChunkUtils {
-
- /* Never constructed. */
- private ChunkUtils() {
- }
-
- /**
- * Checks if we are getting a request to overwrite an existing range of
- * chunk.
- *
- * @param chunkFile - File
- * @param chunkInfo - Buffer to write
- * @return bool
- */
- public static boolean isOverWriteRequested(File chunkFile, ChunkInfo
- chunkInfo) {
-
- if (!chunkFile.exists()) {
- return false;
- }
-
- long offset = chunkInfo.getOffset();
- return offset < chunkFile.length();
- }
-
- /**
- * Overwrite is permitted if an only if the user explicitly asks for it. We
- * permit this iff the key/value pair contains a flag called
- * [OverWriteRequested, true].
- *
- * @param chunkInfo - Chunk info
- * @return true if the user asks for it.
- */
- public static boolean isOverWritePermitted(ChunkInfo chunkInfo) {
- String overWrite = chunkInfo.getMetadata().get(OzoneConsts.CHUNK_OVERWRITE);
- return (overWrite != null) &&
- (!overWrite.isEmpty()) &&
- (Boolean.valueOf(overWrite));
- }
-
- /**
- * Validates chunk data and returns a file object to Chunk File that we are
- * expected to write data to.
- *
- * @param pipeline - pipeline.
- * @param data - container data.
- * @param info - chunk info.
- * @return File
- * @throws StorageContainerException
- */
- public static File validateChunk(Pipeline pipeline, ContainerData data,
- ChunkInfo info) throws StorageContainerException {
-
- Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-
- File chunkFile = getChunkFile(pipeline, data, info);
- if (ChunkUtils.isOverWriteRequested(chunkFile, info)) {
- if (!ChunkUtils.isOverWritePermitted(info)) {
- log.error("Rejecting write chunk request. Chunk overwrite " +
- "without explicit request. {}", info.toString());
- throw new StorageContainerException("Rejecting write chunk request. " +
- "OverWrite flag required." + info.toString(),
- OVERWRITE_FLAG_REQUIRED);
- }
- }
- return chunkFile;
- }
-
- /**
- * Validates that Path to chunk file exists.
- *
- * @param pipeline - Container Info.
- * @param data - Container Data
- * @param info - Chunk info
- * @return - File.
- * @throws StorageContainerException
- */
- public static File getChunkFile(Pipeline pipeline, ContainerData data,
- ChunkInfo info) throws StorageContainerException {
-
- Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
- if (data == null) {
- log.error("Invalid container Name: {}", pipeline.getContainerName());
- throw new StorageContainerException("Unable to find the container Name:" +
- " " +
- pipeline.getContainerName(), CONTAINER_NOT_FOUND);
- }
-
- File dataDir = ContainerUtils.getDataDirectory(data).toFile();
- if (!dataDir.exists()) {
- log.error("Unable to find the data directory: {}", dataDir);
- throw new StorageContainerException("Unable to find the data directory:" +
- " " + dataDir, UNABLE_TO_FIND_DATA_DIR);
- }
-
- return dataDir.toPath().resolve(info.getChunkName()).toFile();
-
- }
-
- /**
- * Writes the data in chunk Info to the specified location in the chunkfile.
- *
- * @param chunkFile - File to write data to.
- * @param chunkInfo - Data stream to write.
- * @param data - The data buffer.
- * @throws StorageContainerException
- */
- public static void writeData(File chunkFile, ChunkInfo chunkInfo,
- byte[] data) throws
- StorageContainerException, ExecutionException, InterruptedException,
- NoSuchAlgorithmException {
-
- Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
- if (data.length != chunkInfo.getLen()) {
- String err = String.format("data array does not match the length " +
- "specified. DataLen: %d Byte Array: %d",
- chunkInfo.getLen(), data.length);
- log.error(err);
- throw new StorageContainerException(err, INVALID_WRITE_SIZE);
- }
-
- AsynchronousFileChannel file = null;
- FileLock lock = null;
-
- try {
- file =
- AsynchronousFileChannel.open(chunkFile.toPath(),
- StandardOpenOption.CREATE,
- StandardOpenOption.WRITE,
- StandardOpenOption.SPARSE,
- StandardOpenOption.SYNC);
- lock = file.lock().get();
- if (chunkInfo.getChecksum() != null &&
- !chunkInfo.getChecksum().isEmpty()) {
- verifyChecksum(chunkInfo, data, log);
- }
- int size = file.write(ByteBuffer.wrap(data), chunkInfo.getOffset()).get();
- if (size != data.length) {
- log.error("Invalid write size found. Size:{} Expected: {} ", size,
- data.length);
- throw new StorageContainerException("Invalid write size found. " +
- "Size: " + size + " Expected: " + data.length, INVALID_WRITE_SIZE);
- }
- } catch (IOException e) {
- throw new StorageContainerException(e, IO_EXCEPTION);
-
- } finally {
- if (lock != null) {
- try {
- lock.release();
- } catch (IOException e) {
- log.error("Unable to release lock ??, Fatal Error.");
- throw new StorageContainerException(e, CONTAINER_INTERNAL_ERROR);
-
- }
- }
- if (file != null) {
- try {
- file.close();
- } catch (IOException e) {
- throw new StorageContainerException("Error closing chunk file",
- e, CONTAINER_INTERNAL_ERROR);
- }
- }
- }
- }
-
- /**
- * Verifies the checksum of a chunk against the data buffer.
- *
- * @param chunkInfo - Chunk Info.
- * @param data - data buffer
- * @param log - log
- * @throws NoSuchAlgorithmException
- * @throws StorageContainerException
- */
- private static void verifyChecksum(ChunkInfo chunkInfo, byte[] data, Logger
- log) throws NoSuchAlgorithmException, StorageContainerException {
- MessageDigest sha = MessageDigest.getInstance(OzoneConsts.FILE_HASH);
- sha.update(data);
- if (!Hex.encodeHexString(sha.digest()).equals(
- chunkInfo.getChecksum())) {
- log.error("Checksum mismatch. Provided: {} , computed: {}",
- chunkInfo.getChecksum(), DigestUtils.sha256Hex(sha.digest()));
- throw new StorageContainerException("Checksum mismatch. Provided: " +
- chunkInfo.getChecksum() + " , computed: " +
- DigestUtils.sha256Hex(sha.digest()), CHECKSUM_MISMATCH);
- }
- }
-
- /**
- * Reads data from an existing chunk file.
- *
- * @param chunkFile - file where data lives.
- * @param data - chunk definition.
- * @return ByteBuffer
- * @throws StorageContainerException
- * @throws ExecutionException
- * @throws InterruptedException
- */
- public static ByteBuffer readData(File chunkFile, ChunkInfo data) throws
- StorageContainerException, ExecutionException, InterruptedException,
- NoSuchAlgorithmException {
- Logger log = LoggerFactory.getLogger(ChunkManagerImpl.class);
-
- if (!chunkFile.exists()) {
- log.error("Unable to find the chunk file. chunk info : {}",
- data.toString());
- throw new StorageContainerException("Unable to find the chunk file. " +
- "chunk info " +
- data.toString(), UNABLE_TO_FIND_CHUNK);
- }
-
- AsynchronousFileChannel file = null;
- FileLock lock = null;
- try {
- file =
- AsynchronousFileChannel.open(chunkFile.toPath(),
- StandardOpenOption.READ);
- lock = file.lock(data.getOffset(), data.getLen(), true).get();
-
- ByteBuffer buf = ByteBuffer.allocate((int) data.getLen());
- file.read(buf, data.getOffset()).get();
-
- if (data.getChecksum() != null && !data.getChecksum().isEmpty()) {
- verifyChecksum(data, buf.array(), log);
- }
-
- return buf;
- } catch (IOException e) {
- throw new StorageContainerException(e, IO_EXCEPTION);
- } finally {
- if (lock != null) {
- try {
- lock.release();
- } catch (IOException e) {
- log.error("I/O error is lock release.");
- }
- }
- if (file != null) {
- IOUtils.closeStream(file);
- }
- }
- }
-
- /**
- * Returns a CreateContainer Response. This call is used by create and delete
- * containers which have null success responses.
- *
- * @param msg Request
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getChunkResponse(ContainerProtos.ContainerCommandRequestProto msg) {
- return ContainerUtils.getContainerResponse(msg);
- }
-
- /**
- * Gets a response to the read chunk calls.
- *
- * @param msg - Msg
- * @param data - Data
- * @param info - Info
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getReadChunkResponse(ContainerProtos.ContainerCommandRequestProto msg,
- byte[] data, ChunkInfo info) {
- Preconditions.checkNotNull(msg);
-
- ContainerProtos.ReadChunkResponseProto.Builder response =
- ContainerProtos.ReadChunkResponseProto.newBuilder();
- response.setChunkData(info.getProtoBufMessage());
- response.setData(ByteString.copyFrom(data));
- response.setPipeline(msg.getReadChunk().getPipeline());
-
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
- .SUCCESS, "");
- builder.setReadChunk(response);
- return builder.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
deleted file mode 100644
index d0886e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerData.java
+++ /dev/null
@@ -1,326 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.util.Time;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.Map;
-import java.util.TreeMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * This class maintains the information about a container in the ozone world.
- * <p>
- * A container is a name, along with metadata- which is a set of key value
- * pair.
- */
-public class ContainerData {
-
- private final String containerName;
- private final Map<String, String> metadata;
- private String dbPath; // Path to Level DB Store.
- // Path to Physical file system where container and checksum are stored.
- private String containerFilePath;
- private String hash;
- private AtomicLong bytesUsed;
- private long maxSize;
- private Long containerID;
- private OzoneProtos.LifeCycleState state;
-
- /**
- * Constructs a ContainerData Object.
- *
- * @param containerName - Name
- */
- public ContainerData(String containerName, Long containerID,
- Configuration conf) {
- this.metadata = new TreeMap<>();
- this.containerName = containerName;
- this.maxSize = conf.getLong(ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_KEY,
- ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT) * OzoneConsts.GB;
- this.bytesUsed = new AtomicLong(0L);
- this.containerID = containerID;
- this.state = OzoneProtos.LifeCycleState.OPEN;
- }
-
- /**
- * Constructs a ContainerData object from ProtoBuf classes.
- *
- * @param protoData - ProtoBuf Message
- * @throws IOException
- */
- public static ContainerData getFromProtBuf(
- ContainerProtos.ContainerData protoData, Configuration conf)
- throws IOException {
- ContainerData data = new ContainerData(protoData.getName(),
- protoData.getContainerID(), conf);
- for (int x = 0; x < protoData.getMetadataCount(); x++) {
- data.addMetadata(protoData.getMetadata(x).getKey(),
- protoData.getMetadata(x).getValue());
- }
-
- if (protoData.hasContainerPath()) {
- data.setContainerPath(protoData.getContainerPath());
- }
-
- if (protoData.hasDbPath()) {
- data.setDBPath(protoData.getDbPath());
- }
-
- if (protoData.hasState()) {
- data.setState(protoData.getState());
- }
-
- if(protoData.hasHash()) {
- data.setHash(protoData.getHash());
- }
-
- if (protoData.hasBytesUsed()) {
- data.setBytesUsed(protoData.getBytesUsed());
- }
-
- if (protoData.hasSize()) {
- data.setMaxSize(protoData.getSize());
- }
- return data;
- }
-
- /**
- * Returns a ProtoBuf Message from ContainerData.
- *
- * @return Protocol Buffer Message
- */
- public ContainerProtos.ContainerData getProtoBufMessage() {
- ContainerProtos.ContainerData.Builder builder = ContainerProtos
- .ContainerData.newBuilder();
- builder.setName(this.getContainerName());
- builder.setContainerID(this.getContainerID());
-
- if (this.getDBPath() != null) {
- builder.setDbPath(this.getDBPath());
- }
-
- if (this.getHash() != null) {
- builder.setHash(this.getHash());
- }
-
- if (this.getContainerPath() != null) {
- builder.setContainerPath(this.getContainerPath());
- }
-
- builder.setState(this.getState());
-
- for (Map.Entry<String, String> entry : metadata.entrySet()) {
- OzoneProtos.KeyValue.Builder keyValBuilder =
- OzoneProtos.KeyValue.newBuilder();
- builder.addMetadata(keyValBuilder.setKey(entry.getKey())
- .setValue(entry.getValue()).build());
- }
-
- if (this.getBytesUsed() >= 0) {
- builder.setBytesUsed(this.getBytesUsed());
- }
-
- if (this.getKeyCount() >= 0) {
- builder.setKeyCount(this.getKeyCount());
- }
-
- if (this.getMaxSize() >= 0) {
- builder.setSize(this.getMaxSize());
- }
-
- return builder.build();
- }
-
- /**
- * Returns the name of the container.
- *
- * @return - name
- */
- public String getContainerName() {
- return containerName;
- }
-
- /**
- * Adds metadata.
- */
- public void addMetadata(String key, String value) throws IOException {
- synchronized (this.metadata) {
- if (this.metadata.containsKey(key)) {
- throw new IOException("This key already exists. Key " + key);
- }
- metadata.put(key, value);
- }
- }
-
- /**
- * Returns all metadata.
- */
- public Map<String, String> getAllMetadata() {
- synchronized (this.metadata) {
- return Collections.unmodifiableMap(this.metadata);
- }
- }
-
- /**
- * Returns value of a key.
- */
- public String getValue(String key) {
- synchronized (this.metadata) {
- return metadata.get(key);
- }
- }
-
- /**
- * Deletes a metadata entry from the map.
- *
- * @param key - Key
- */
- public void deleteKey(String key) {
- synchronized (this.metadata) {
- metadata.remove(key);
- }
- }
-
- /**
- * Returns path.
- *
- * @return - path
- */
- public String getDBPath() {
- return dbPath;
- }
-
- /**
- * Sets path.
- *
- * @param path - String.
- */
- public void setDBPath(String path) {
- this.dbPath = path;
- }
-
- /**
- * This function serves as the generic key for ContainerCache class. Both
- * ContainerData and ContainerKeyData overrides this function to appropriately
- * return the right name that can be used in ContainerCache.
- *
- * @return String Name.
- */
- public String getName() {
- return getContainerName();
- }
-
- /**
- * Get container file path.
- * @return - Physical path where container file and checksum is stored.
- */
- public String getContainerPath() {
- return containerFilePath;
- }
-
- /**
- * Set container Path.
- * @param containerPath - File path.
- */
- public void setContainerPath(String containerPath) {
- this.containerFilePath = containerPath;
- }
-
- /**
- * Get container ID.
- * @return - container ID.
- */
- public synchronized Long getContainerID() {
- return containerID;
- }
-
- public synchronized void setState(OzoneProtos.LifeCycleState state) {
- this.state = state;
- }
-
- public synchronized OzoneProtos.LifeCycleState getState() {
- return this.state;
- }
-
- /**
- * checks if the container is open.
- * @return - boolean
- */
- public synchronized boolean isOpen() {
- return OzoneProtos.LifeCycleState.OPEN == state;
- }
-
- /**
- * Marks this container as closed.
- */
- public synchronized void closeContainer() {
- // TODO: closed or closing here
- setState(OzoneProtos.LifeCycleState.CLOSED);
-
- // Some thing brain dead for now. name + Time stamp of when we get the close
- // container message.
- setHash(DigestUtils.sha256Hex(this.getContainerName() +
- Long.toString(Time.monotonicNow())));
- }
-
- /**
- * Final hash for this container.
- * @return - Hash
- */
- public String getHash() {
- return hash;
- }
-
- public void setHash(String hash) {
- this.hash = hash;
- }
-
- public void setMaxSize(long maxSize) {
- this.maxSize = maxSize;
- }
-
- public long getMaxSize() {
- return maxSize;
- }
-
- public long getKeyCount() {
- return metadata.size();
- }
-
- public void setBytesUsed(long used) {
- this.bytesUsed.set(used);
- }
-
- public long addBytesUsed(long delta) {
- return this.bytesUsed.addAndGet(delta);
- }
-
- public long getBytesUsed() {
- return bytesUsed.get();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/pom.xml b/hadoop-cblock/server/pom.xml
new file mode 100644
index 0000000..17af85d
--- /dev/null
+++ b/hadoop-cblock/server/pom.xml
@@ -0,0 +1,169 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-cblock</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ </parent>
+ <artifactId>hadoop-cblock-server</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <description>Apache Hadoop CBlock Server</description>
+ <name>Apache Hadoop CBlock Server</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <hadoop.component>cblock</hadoop.component>
+ <is.hadoop.component>true</is.hadoop.component>
+ </properties>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-server-framework</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdsl-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-ozone-integration-test</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.jscsi</groupId>
+ <artifactId>target</artifactId>
+ <version>2.6.0</version>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.kubernetes</groupId>
+ <artifactId>client-java</artifactId>
+ <version>1.0.0-beta1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.swagger</groupId>
+ <artifactId>swagger-annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.github.stefanbirkner</groupId>
+ <artifactId>system-rules</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+
+ </dependencies>
+
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/test/resources/dynamicprovisioner/expected1-pv.json</exclude>
+ <exclude>src/test/resources/dynamicprovisioner/input1-pvc.json</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-maven-plugins</artifactId>
+ <executions>
+ <execution>
+ <id>compile-protoc</id>
+ <goals>
+ <goal>protoc</goal>
+ </goals>
+ <configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
+ <imports>
+ <param>
+ ${basedir}/../../hadoop-common-project/hadoop-common/src/main/proto
+ </param>
+ <param>
+ ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/
+ </param>
+ <param>
+ ${basedir}/../../hadoop-hdfs-project/hadoop-hdfs/src/main/proto/
+ </param>
+ <param>
+ ${basedir}/../../hadoop-hdsl/common/src/main/proto/
+ </param>
+ <param>${basedir}/src/main/proto</param>
+ </imports>
+ <source>
+ <directory>${basedir}/src/main/proto</directory>
+ <includes>
+ <include>CBlockClientServerProtocol.proto</include>
+ <include>CBlockServiceProtocol.proto</include>
+ </includes>
+ </source>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
new file mode 100644
index 0000000..ec5d7d2
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockConfigKeys.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import static java.lang.Thread.NORM_PRIORITY;
+
+/**
+ * This class contains constants for configuration keys used in CBlock.
+ */
+public final class CBlockConfigKeys {
+
+ public static final String DFS_CBLOCK_SERVICERPC_ADDRESS_KEY =
+ "dfs.cblock.servicerpc-address";
+ public static final int DFS_CBLOCK_SERVICERPC_PORT_DEFAULT =
+ 9810;
+ public static final String DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT =
+ "0.0.0.0";
+
+ public static final String DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY =
+ "dfs.cblock.jscsi-address";
+
+ //The port on CBlockManager node for jSCSI to ask
+ public static final String DFS_CBLOCK_JSCSI_PORT_KEY =
+ "dfs.cblock.jscsi.port";
+ public static final int DFS_CBLOCK_JSCSI_PORT_DEFAULT =
+ 9811;
+
+ public static final String DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY =
+ "dfs.cblock.service.rpc-bind-host";
+ public static final String DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY =
+ "dfs.cblock.jscsi.rpc-bind-host";
+
+ // default block size is 4KB
+ public static final int DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT =
+ 4096;
+
+ public static final String DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY =
+ "dfs.cblock.service.handler.count";
+ public static final int DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT = 10;
+
+ public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY =
+ "dfs.cblock.service.leveldb.path";
+ //TODO : find a better place
+ public static final String DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT =
+ "/tmp/cblock_levelDB.dat";
+
+
+ public static final String DFS_CBLOCK_DISK_CACHE_PATH_KEY =
+ "dfs.cblock.disk.cache.path";
+ public static final String DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT =
+ "/tmp/cblockCacheDB";
+ /**
+ * Setting this flag to true makes the block layer compute a sha256 hash of
+ * the data and log that information along with block ID. This is very
+ * useful for doing trace based simulation of various workloads. Since it is
+ * computing a hash for each block this could be expensive, hence default
+ * is false.
+ */
+ public static final String DFS_CBLOCK_TRACE_IO = "dfs.cblock.trace.io";
+ public static final boolean DFS_CBLOCK_TRACE_IO_DEFAULT = false;
+
+ public static final String DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO =
+ "dfs.cblock.short.circuit.io";
+ public static final boolean DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT =
+ false;
+
+ /**
+ * Cache size in 1000s of entries. 256 indicates 256 * 1024.
+ */
+ public static final String DFS_CBLOCK_CACHE_QUEUE_SIZE_KB =
+ "dfs.cblock.cache.queue.size.in.kb";
+ public static final int DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT = 256;
+
+ /**
+ * Minimum Number of threads that cache pool will use for background I/O.
+ */
+ public static final String DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE =
+ "dfs.cblock.cache.core.min.pool.size";
+ public static final int DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT = 16;
+
+ /**
+ * Maximum Number of threads that cache pool will use for background I/O.
+ */
+
+ public static final String DFS_CBLOCK_CACHE_MAX_POOL_SIZE =
+ "dfs.cblock.cache.max.pool.size";
+ public static final int DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT = 256;
+
+ /**
+ * Number of seconds to keep the Thread alive when it is idle.
+ */
+ public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE =
+ "dfs.cblock.cache.keep.alive";
+ public static final String DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT = "60s";
+
+ /**
+ * Priority of cache flusher thread, affecting the relative performance of
+ * write and read.
+ */
+ public static final String DFS_CBLOCK_CACHE_THREAD_PRIORITY =
+ "dfs.cblock.cache.thread.priority";
+ public static final int DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT =
+ NORM_PRIORITY;
+
+ /**
+ * Block Buffer size in terms of blockID entries, 512 means 512 blockIDs.
+ */
+ public static final String DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE =
+ "dfs.cblock.cache.block.buffer.size";
+ public static final int DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT = 512;
+
+ public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL =
+ "dfs.cblock.block.buffer.flush.interval";
+ public static final String DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT =
+ "60s";
+
+ // jscsi server settings
+ public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY =
+ "dfs.cblock.jscsi.server.address";
+ public static final String DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT =
+ "0.0.0.0";
+ public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY =
+ "dfs.cblock.jscsi.cblock.server.address";
+ public static final String DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT =
+ "127.0.0.1";
+
+ // to what address cblock server should talk to scm?
+ public static final String DFS_CBLOCK_SCM_IPADDRESS_KEY =
+ "dfs.cblock.scm.ipaddress";
+ public static final String DFS_CBLOCK_SCM_IPADDRESS_DEFAULT =
+ "127.0.0.1";
+ public static final String DFS_CBLOCK_SCM_PORT_KEY =
+ "dfs.cblock.scm.port";
+ public static final int DFS_CBLOCK_SCM_PORT_DEFAULT = 9860;
+
+ public static final String DFS_CBLOCK_CONTAINER_SIZE_GB_KEY =
+ "dfs.cblock.container.size.gb";
+ public static final int DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT =
+ 5;
+
+ // LevelDB cache file uses an off-heap cache in LevelDB of 256 MB.
+ public static final String DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY =
+ "dfs.cblock.cache.leveldb.cache.size.mb";
+ public static final int DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT = 256;
+
+ /**
+ * Cache does an best case attempt to write a block to a container.
+ * At some point of time, we will need to handle the case where we did try
+ * 64K times and is till not able to write to the container.
+ *
+ * TODO: We will need cBlock Server to allow us to do a remapping of the
+ * block location in case of failures, at that point we should reduce the
+ * retry count to a more normal number. This is approximately 18 hours of
+ * retry.
+ */
+ public static final String DFS_CBLOCK_CACHE_MAX_RETRY_KEY =
+ "dfs.cblock.cache.max.retry";
+ public static final int DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT =
+ 64 * 1024;
+
+ /**
+ * Cblock CLI configs.
+ */
+ public static final String DFS_CBLOCK_MANAGER_POOL_SIZE =
+ "dfs.cblock.manager.pool.size";
+ public static final int DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT = 16;
+
+ /**
+ * currently the largest supported volume is about 8TB, which might take
+ * > 20 seconds to finish creating containers. thus set timeout to 30 sec.
+ */
+ public static final String DFS_CBLOCK_RPC_TIMEOUT =
+ "dfs.cblock.rpc.timeout";
+ public static final String DFS_CBLOCK_RPC_TIMEOUT_DEFAULT = "300s";
+
+ public static final String DFS_CBLOCK_ISCSI_ADVERTISED_IP =
+ "dfs.cblock.iscsi.advertised.ip";
+
+ public static final String DFS_CBLOCK_ISCSI_ADVERTISED_PORT =
+ "dfs.cblock.iscsi.advertised.port";
+
+ public static final int DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT = 3260;
+
+
+ public static final String
+ DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED
+ = "dfs.cblock.kubernetes.dynamic-provisioner.enabled";
+
+ public static final boolean
+ DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT = false;
+
+ public static final String
+ DFS_CBLOCK_KUBERNETES_CBLOCK_USER =
+ "dfs.cblock.kubernetes.cblock-user";
+
+ public static final String
+ DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT =
+ "iqn.2001-04.org.apache.hadoop";
+
+ public static final String
+ DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY =
+ "dfs.cblock.kubernetes.configfile";
+
+ private CBlockConfigKeys() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
new file mode 100644
index 0000000..788d01e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
@@ -0,0 +1,426 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.BlockingService;
+import org.apache.hadoop.cblock.kubernetes.DynamicProvisioner;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.cblock.protocolPB
+ .CBlockClientServerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.cblock.protocolPB
+ .CBlockServiceProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.client.ContainerOperationClient;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.cblock.storage.StorageManager;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.protocolPB
+ .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.utils.LevelDBStore;
+
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServerRpcAddr;
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr;
+import static org.apache.hadoop.ozone.web.util.ServerUtils
+ .updateRPCListenAddress;
+import org.iq80.leveldb.DBIterator;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SCM_IPADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SCM_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SCM_PORT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT;
+
+
+/**
+ * The main entry point of CBlock operations, ALL the CBlock operations
+ * will go through this class. But NOTE that:
+ *
+ * volume operations (create/
+ * delete/info) are:
+ * client -> CBlockManager -> StorageManager -> CBlock client
+ *
+ * IO operations (put/get block) are;
+ * client -> CBlock client -> container
+ *
+ */
+public class CBlockManager implements CBlockServiceProtocol,
+ CBlockClientProtocol {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CBlockManager.class);
+
+ private final RPC.Server cblockService;
+ private final RPC.Server cblockServer;
+
+ private final StorageManager storageManager;
+
+ private final LevelDBStore levelDBStore;
+ private final String dbPath;
+
+ private final DynamicProvisioner kubernetesDynamicProvisioner;
+
+ private Charset encoding = Charset.forName("UTF-8");
+
+ public CBlockManager(OzoneConfiguration conf,
+ ScmClient storageClient) throws IOException {
+ // Fix the cBlockManagerId generattion code here. Should support
+ // cBlockManager --init command which will generate a cBlockManagerId and
+ // persist it locally.
+ storageManager =
+ new StorageManager(storageClient, conf, "CBLOCK");
+
+ dbPath = conf.getTrimmed(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY,
+ DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT);
+ levelDBStore = new LevelDBStore(new File(dbPath), true);
+ LOG.info("Try to load exising volume information");
+ readFromPersistentStore();
+
+ RPC.setProtocolEngine(conf, CBlockServiceProtocolPB.class,
+ ProtobufRpcEngine.class);
+ RPC.setProtocolEngine(conf, CBlockClientServerProtocolPB.class,
+ ProtobufRpcEngine.class);
+ // start service for client command-to-cblock server service
+ InetSocketAddress serviceRpcAddr =
+ getCblockServiceRpcAddr(conf);
+ BlockingService cblockProto =
+ CBlockServiceProtocolProtos
+ .CBlockServiceProtocolService
+ .newReflectiveBlockingService(
+ new CBlockServiceProtocolServerSideTranslatorPB(this)
+ );
+ cblockService = startRpcServer(conf, CBlockServiceProtocolPB.class,
+ cblockProto, serviceRpcAddr,
+ DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY,
+ DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
+ DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
+ InetSocketAddress cblockServiceRpcAddress =
+ updateRPCListenAddress(conf,
+ DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, serviceRpcAddr, cblockService);
+ LOG.info("CBlock manager listening for client commands on: {}",
+ cblockServiceRpcAddress);
+ // now start service for cblock client-to-cblock server communication
+
+ InetSocketAddress serverRpcAddr =
+ getCblockServerRpcAddr(conf);
+ BlockingService serverProto =
+ CBlockClientServerProtocolProtos
+ .CBlockClientServerProtocolService
+ .newReflectiveBlockingService(
+ new CBlockClientServerProtocolServerSideTranslatorPB(this)
+ );
+ cblockServer = startRpcServer(
+ conf, CBlockClientServerProtocolPB.class,
+ serverProto, serverRpcAddr,
+ DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY,
+ DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
+ DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
+ InetSocketAddress cblockServerRpcAddress =
+ updateRPCListenAddress(conf,
+ DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, serverRpcAddr, cblockServer);
+ LOG.info("CBlock server listening for client commands on: {}",
+ cblockServerRpcAddress);
+
+ if (conf.getBoolean(DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED,
+ DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT)) {
+
+ kubernetesDynamicProvisioner =
+ new DynamicProvisioner(conf, storageManager);
+ kubernetesDynamicProvisioner.init();
+
+ } else {
+ kubernetesDynamicProvisioner = null;
+ }
+ }
+
+ public void start() {
+ cblockService.start();
+ cblockServer.start();
+ if (kubernetesDynamicProvisioner != null) {
+ kubernetesDynamicProvisioner.start();
+ }
+ LOG.info("CBlock manager started!");
+ }
+
+ public void stop() {
+ cblockService.stop();
+ cblockServer.stop();
+ if (kubernetesDynamicProvisioner != null) {
+ kubernetesDynamicProvisioner.stop();
+ }
+ }
+
+ public void join() {
+ try {
+ cblockService.join();
+ cblockServer.join();
+ } catch (InterruptedException e) {
+ LOG.error("Interrupted during join");
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Starts an RPC server, if configured.
+ *
+ * @param conf configuration
+ * @param protocol RPC protocol provided by RPC server
+ * @param instance RPC protocol implementation instance
+ * @param addr configured address of RPC server
+ * @param bindHostKey configuration key for setting explicit bind host. If
+ * the property is not configured, then the bind host is taken from addr.
+ * @param handlerCountKey configuration key for RPC server handler count
+ * @param handlerCountDefault default RPC server handler count if unconfigured
+ * @return RPC server, or null if addr is null
+ * @throws IOException if there is an I/O error while creating RPC server
+ */
+ private static RPC.Server startRpcServer(OzoneConfiguration conf,
+ Class<?> protocol, BlockingService instance,
+ InetSocketAddress addr, String bindHostKey,
+ String handlerCountKey, int handlerCountDefault) throws IOException {
+ if (addr == null) {
+ return null;
+ }
+ String bindHost = conf.getTrimmed(bindHostKey);
+ if (bindHost == null || bindHost.isEmpty()) {
+ bindHost = addr.getHostName();
+ }
+ int numHandlers = conf.getInt(handlerCountKey, handlerCountDefault);
+ RPC.Server rpcServer = new RPC.Builder(conf)
+ .setProtocol(protocol)
+ .setInstance(instance)
+ .setBindAddress(bindHost)
+ .setPort(addr.getPort())
+ .setNumHandlers(numHandlers)
+ .setVerbose(false)
+ .setSecretManager(null)
+ .build();
+ return rpcServer;
+ }
+
+ @Override
+ public synchronized MountVolumeResponse mountVolume(
+ String userName, String volumeName) throws IOException {
+ return storageManager.isVolumeValid(userName, volumeName);
+ }
+
+ @Override
+ public List<VolumeInfo> listVolumes() throws IOException {
+ return listVolume(null);
+ }
+
+ @Override
+ public synchronized void createVolume(String userName, String volumeName,
+ long volumeSize, int blockSize) throws IOException {
+ LOG.info("Create volume received: userName: {} volumeName: {} " +
+ "volumeSize: {} blockSize: {}", userName, volumeName,
+ volumeSize, blockSize);
+ // It is important to create in-memory representation of the
+ // volume first, then writes to persistent storage (levelDB)
+ // such that it is guaranteed that when there is an entry in
+ // levelDB, the volume is allocated. (more like a UNDO log fashion)
+ // TODO: what if creation failed? we allocated containers but lost
+ // the reference to the volume and all it's containers. How to release
+ // the containers?
+ storageManager.createVolume(userName, volumeName, volumeSize, blockSize);
+ VolumeDescriptor volume = storageManager.getVolume(userName, volumeName);
+ if (volume == null) {
+ throw new IOException("Volume creation failed!");
+ }
+ String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
+ writeToPersistentStore(volumeKey.getBytes(encoding),
+ volume.toProtobuf().toByteArray());
+ }
+
+ @Override
+ public synchronized void deleteVolume(String userName,
+ String volumeName, boolean force) throws IOException {
+ LOG.info("Delete volume received: volume: {} {} ", volumeName, force);
+ storageManager.deleteVolume(userName, volumeName, force);
+ // being here means volume is successfully deleted now
+ String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
+ removeFromPersistentStore(volumeKey.getBytes(encoding));
+ }
+
+ // No need to synchronize on the following three methods, since write and
+ // remove's caller are synchronized. read's caller is the constructor and
+ // no other method call can happen at that time.
+ @VisibleForTesting
+ public void writeToPersistentStore(byte[] key, byte[] value) {
+ levelDBStore.put(key, value);
+ }
+
+ @VisibleForTesting
+ public void removeFromPersistentStore(byte[] key) {
+ levelDBStore.delete(key);
+ }
+
+ public void readFromPersistentStore() throws IOException {
+ try (DBIterator iter = levelDBStore.getIterator()) {
+ iter.seekToFirst();
+ while (iter.hasNext()) {
+ Map.Entry<byte[], byte[]> entry = iter.next();
+ String volumeKey = new String(entry.getKey(), encoding);
+ try {
+ VolumeDescriptor volumeDescriptor =
+ VolumeDescriptor.fromProtobuf(entry.getValue());
+ storageManager.addVolume(volumeDescriptor);
+ } catch (IOException e) {
+ LOG.error("Loading volume " + volumeKey + " error " + e);
+ }
+ }
+ }
+ }
+
+ @Override
+ public synchronized VolumeInfo infoVolume(String userName, String volumeName
+ ) throws IOException {
+ LOG.info("Info volume received: volume: {}", volumeName);
+ return storageManager.infoVolume(userName, volumeName);
+ }
+
+ @VisibleForTesting
+ public synchronized List<VolumeDescriptor> getAllVolumes() {
+ return storageManager.getAllVolume(null);
+ }
+
+ public synchronized List<VolumeDescriptor> getAllVolumes(String userName) {
+ return storageManager.getAllVolume(userName);
+ }
+
+ public synchronized void close() {
+ try {
+ levelDBStore.close();
+ } catch (IOException e) {
+ LOG.error("Error when closing levelDB " + e);
+ }
+ }
+
+ public synchronized void clean() {
+ try {
+ levelDBStore.close();
+ levelDBStore.destroy();
+ } catch (IOException e) {
+ LOG.error("Error when deleting levelDB " + e);
+ }
+ }
+
+ @Override
+ public synchronized List<VolumeInfo> listVolume(String userName)
+ throws IOException {
+ ArrayList<VolumeInfo> response = new ArrayList<>();
+ List<VolumeDescriptor> allVolumes =
+ storageManager.getAllVolume(userName);
+ for (VolumeDescriptor volume : allVolumes) {
+ VolumeInfo info =
+ new VolumeInfo(volume.getUserName(), volume.getVolumeName(),
+ volume.getVolumeSize(), volume.getBlockSize());
+ response.add(info);
+ }
+ return response;
+ }
+
+ public static void main(String[] args) throws Exception {
+ long version = RPC.getProtocolVersion(
+ StorageContainerLocationProtocolPB.class);
+ CblockUtils.activateConfigs();
+ OzoneConfiguration ozoneConf = new OzoneConfiguration();
+ String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
+ DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
+ int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
+ DFS_CBLOCK_SCM_PORT_DEFAULT);
+ int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
+ DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
+ ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
+ InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
+
+ ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
+ OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
+ LOG.info(
+ "Creating StorageContainerLocationProtocol RPC client with address {}",
+ address);
+ RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
+ ProtobufRpcEngine.class);
+ StorageContainerLocationProtocolClientSideTranslatorPB client =
+ new StorageContainerLocationProtocolClientSideTranslatorPB(
+ RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
+ address, UserGroupInformation.getCurrentUser(), ozoneConf,
+ NetUtils.getDefaultSocketFactory(ozoneConf),
+ Client.getRpcTimeout(ozoneConf)));
+ ScmClient storageClient = new ContainerOperationClient(
+ client, new XceiverClientManager(ozoneConf));
+ CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
+ cbm.start();
+ cbm.join();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
new file mode 100644
index 0000000..99ffde0
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/CblockUtils.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.cblock;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+
+import com.google.common.base.Optional;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSI_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
+import static org.apache.hadoop.hdsl.HdslUtils.getHostNameFromConfigKeys;
+import static org.apache.hadoop.hdsl.HdslUtils.getPortNumberFromConfigKeys;
+
+/**
+ * Generic stateless utility functions for CBlock components.
+ */
+public class CblockUtils {
+
+ private CblockUtils() {
+ }
+
+ /**
+ * Retrieve the socket address that is used by CBlock Service.
+ *
+ * @param conf
+ * @return Target InetSocketAddress for the CBlock Service endpoint.
+ */
+ public static InetSocketAddress getCblockServiceRpcAddr(Configuration conf) {
+ final Optional<String> host =
+ getHostNameFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
+
+ // If no port number is specified then we'll just try the defaultBindPort.
+ final Optional<Integer> port =
+ getPortNumberFromConfigKeys(conf, DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
+ host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port
+ .or(DFS_CBLOCK_SERVICERPC_PORT_DEFAULT));
+ }
+
+ /**
+ * Retrieve the socket address that is used by CBlock Server.
+ *
+ * @param conf
+ * @return Target InetSocketAddress for the CBlock Server endpoint.
+ */
+ public static InetSocketAddress getCblockServerRpcAddr(Configuration conf) {
+ final Optional<String> host =
+ getHostNameFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
+
+ // If no port number is specified then we'll just try the defaultBindPort.
+ final Optional<Integer> port =
+ getPortNumberFromConfigKeys(conf, DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
+
+ return NetUtils.createSocketAddr(
+ host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" + port
+ .or(DFS_CBLOCK_JSCSI_PORT_DEFAULT));
+ }
+
+ /**
+ * Parse size with size prefix string and return in bytes.
+ *
+ */
+ public static long parseSize(String volumeSizeArgs) throws IOException {
+ long multiplier = 1;
+
+ Pattern p = Pattern.compile("([0-9]+)([a-zA-Z]+)");
+ Matcher m = p.matcher(volumeSizeArgs);
+
+ if (!m.find()) {
+ throw new IOException("Invalid volume size args " + volumeSizeArgs);
+ }
+
+ int size = Integer.parseInt(m.group(1));
+ String s = m.group(2);
+
+ if (s.equalsIgnoreCase("MB") ||
+ s.equalsIgnoreCase("Mi")) {
+ multiplier = 1024L * 1024;
+ } else if (s.equalsIgnoreCase("GB") ||
+ s.equalsIgnoreCase("Gi")) {
+ multiplier = 1024L * 1024 * 1024;
+ } else if (s.equalsIgnoreCase("TB") ||
+ s.equalsIgnoreCase("Ti")) {
+ multiplier = 1024L * 1024 * 1024 * 1024;
+ } else {
+ throw new IOException("Invalid volume size args " + volumeSizeArgs);
+ }
+ return size * multiplier;
+ }
+
+ public static void activateConfigs(){
+ Configuration.addDefaultResource("hdfs-default.xml");
+ Configuration.addDefaultResource("hdfs-site.xml");
+ Configuration.addDefaultResource("ozone-default.xml");
+ Configuration.addDefaultResource("ozone-site.xml");
+ Configuration.addDefaultResource("cblock-default.xml");
+ Configuration.addDefaultResource("cblock-site.xml");
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..ec38c9c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * The client side implement of CBlockServiceProtocol.
+ */
+@InterfaceAudience.Private
+public final class CBlockServiceProtocolClientSideTranslatorPB
+ implements CBlockServiceProtocol, ProtocolTranslator, Closeable {
+
+ private final CBlockServiceProtocolPB rpcProxy;
+
+ public CBlockServiceProtocolClientSideTranslatorPB(
+ CBlockServiceProtocolPB rpcProxy) {
+ this.rpcProxy = rpcProxy;
+ }
+
+ @Override
+ public void close() throws IOException {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ @Override
+ public void createVolume(String userName, String volumeName,
+ long volumeSize, int blockSize) throws IOException {
+ CBlockServiceProtocolProtos.CreateVolumeRequestProto.Builder req =
+ CBlockServiceProtocolProtos.CreateVolumeRequestProto.newBuilder();
+ req.setUserName(userName);
+ req.setVolumeName(volumeName);
+ req.setVolumeSize(volumeSize);
+ req.setBlockSize(blockSize);
+ try {
+ rpcProxy.createVolume(null, req.build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public void deleteVolume(String userName, String volumeName, boolean force)
+ throws IOException {
+ CBlockServiceProtocolProtos.DeleteVolumeRequestProto.Builder req =
+ CBlockServiceProtocolProtos.DeleteVolumeRequestProto.newBuilder();
+ req.setUserName(userName);
+ req.setVolumeName(volumeName);
+ req.setForce(force);
+ try {
+ rpcProxy.deleteVolume(null, req.build());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+
+ @Override
+ public VolumeInfo infoVolume(String userName, String volumeName)
+ throws IOException {
+ CBlockServiceProtocolProtos.InfoVolumeRequestProto.Builder req =
+ CBlockServiceProtocolProtos.InfoVolumeRequestProto.newBuilder();
+ req.setUserName(userName);
+ req.setVolumeName(volumeName);
+ try {
+ CBlockServiceProtocolProtos.InfoVolumeResponseProto resp =
+ rpcProxy.infoVolume(null, req.build());
+ return new VolumeInfo(resp.getVolumeInfo().getUserName(),
+ resp.getVolumeInfo().getVolumeName(),
+ resp.getVolumeInfo().getVolumeSize(),
+ resp.getVolumeInfo().getBlockSize(),
+ resp.getVolumeInfo().getUsage());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public List<VolumeInfo> listVolume(String userName) throws IOException {
+ CBlockServiceProtocolProtos.ListVolumeRequestProto.Builder req =
+ CBlockServiceProtocolProtos.ListVolumeRequestProto.newBuilder();
+ if (userName != null) {
+ req.setUserName(userName);
+ }
+ try {
+ CBlockServiceProtocolProtos.ListVolumeResponseProto resp =
+ rpcProxy.listVolume(null, req.build());
+ List<VolumeInfo> respList = new ArrayList<>();
+ for (CBlockServiceProtocolProtos.VolumeInfoProto entry :
+ resp.getVolumeEntryList()) {
+ VolumeInfo volumeInfo = new VolumeInfo(
+ entry.getUserName(), entry.getVolumeName(), entry.getVolumeSize(),
+ entry.getBlockSize());
+ respList.add(volumeInfo);
+ }
+ return respList;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ } catch (Exception e) {
+ throw new IOException("got" + e.getCause() + " " + e.getMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
new file mode 100644
index 0000000..4a3878a
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.retry.RetryPolicies;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+
+import static org.apache.hadoop.cblock.CblockUtils.getCblockServiceRpcAddr;
+
+/**
+ * Implementation of client used by CBlock command line tool.
+ */
+public class CBlockVolumeClient {
+ private final CBlockServiceProtocolClientSideTranslatorPB cblockClient;
+
+ public CBlockVolumeClient(OzoneConfiguration conf) throws IOException {
+ this(conf, null);
+ }
+
+ public CBlockVolumeClient(OzoneConfiguration conf,
+ InetSocketAddress serverAddress) throws IOException {
+ InetSocketAddress address = serverAddress != null ? serverAddress :
+ getCblockServiceRpcAddr(conf);
+ long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class);
+ int rpcTimeout = Math.toIntExact(
+ conf.getTimeDuration(CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT,
+ CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT_DEFAULT,
+ TimeUnit.MILLISECONDS));
+ cblockClient = new CBlockServiceProtocolClientSideTranslatorPB(
+ RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version,
+ address, UserGroupInformation.getCurrentUser(), conf,
+ NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies
+ .retryUpToMaximumCountWithFixedSleep(
+ 300, 1, TimeUnit.SECONDS)).getProxy());
+ }
+
+ public void createVolume(String userName, String volumeName,
+ long volumeSize, int blockSize) throws IOException {
+ cblockClient.createVolume(userName, volumeName,
+ volumeSize, blockSize);
+ }
+
+ public void deleteVolume(String userName, String volumeName, boolean force)
+ throws IOException {
+ cblockClient.deleteVolume(userName, volumeName, force);
+ }
+
+ public VolumeInfo infoVolume(String userName, String volumeName)
+ throws IOException {
+ return cblockClient.infoVolume(userName, volumeName);
+ }
+
+ public List<VolumeInfo> listVolume(String userName)
+ throws IOException {
+ return cblockClient.listVolume(userName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
new file mode 100644
index 0000000..761b71e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/client/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.client;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
new file mode 100644
index 0000000..8f6b82b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.exception;
+
+import java.io.IOException;
+
+/**
+ * The exception class used in CBlock.
+ */
+public class CBlockException extends IOException {
+ public CBlockException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
new file mode 100644
index 0000000..268b8cb
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.exception;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
new file mode 100644
index 0000000..04fe3a4
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+
+/**
+ * The blockWriter task.
+ */
+public class BlockWriterTask implements Runnable {
+ private final LogicalBlock block;
+ private int tryCount;
+ private final ContainerCacheFlusher flusher;
+ private final String dbPath;
+ private final String fileName;
+ private final int maxRetryCount;
+
+ /**
+ * Constructs a BlockWriterTask.
+ *
+ * @param block - Block Information.
+ * @param flusher - ContainerCacheFlusher.
+ */
+ public BlockWriterTask(LogicalBlock block, ContainerCacheFlusher flusher,
+ String dbPath, int tryCount, String fileName, int maxRetryCount) {
+ this.block = block;
+ this.flusher = flusher;
+ this.dbPath = dbPath;
+ this.tryCount = tryCount;
+ this.fileName = fileName;
+ this.maxRetryCount = maxRetryCount;
+ }
+
+ /**
+ * When an object implementing interface <code>Runnable</code> is used
+ * to create a thread, starting the thread causes the object's
+ * <code>run</code> method to be called in that separately executing
+ * thread.
+ * <p>
+ * The general contract of the method <code>run</code> is that it may
+ * take any action whatsoever.
+ *
+ * @see Thread#run()
+ */
+ @Override
+ public void run() {
+ String containerName = null;
+ XceiverClientSpi client = null;
+ LevelDBStore levelDBStore = null;
+ String traceID = flusher.getTraceID(new File(dbPath), block.getBlockID());
+ flusher.getLOG().debug(
+ "Writing block to remote. block ID: {}", block.getBlockID());
+ try {
+ incTryCount();
+ Pipeline pipeline = flusher.getPipeline(this.dbPath, block.getBlockID());
+ client = flusher.getXceiverClientManager().acquireClient(pipeline);
+ containerName = pipeline.getContainerName();
+ byte[] keybuf = Longs.toByteArray(block.getBlockID());
+ byte[] data;
+ long startTime = Time.monotonicNow();
+ levelDBStore = flusher.getCacheDB(this.dbPath);
+ data = levelDBStore.get(keybuf);
+ Preconditions.checkNotNull(data);
+ long endTime = Time.monotonicNow();
+ Preconditions.checkState(data.length > 0, "Block data is zero length");
+ startTime = Time.monotonicNow();
+ ContainerProtocolCalls.writeSmallFile(client, containerName,
+ Long.toString(block.getBlockID()), data, traceID);
+ endTime = Time.monotonicNow();
+ flusher.getTargetMetrics().updateContainerWriteLatency(
+ endTime - startTime);
+ flusher.getLOG().debug("Time taken for Write Small File : {} ms",
+ endTime - startTime);
+
+ flusher.incrementRemoteIO();
+
+ } catch (Exception ex) {
+ flusher.getLOG().error("Writing of block:{} failed, We have attempted " +
+ "to write this block {} times to the container {}.Trace ID:{}",
+ block.getBlockID(), this.getTryCount(), containerName, traceID, ex);
+ writeRetryBlock(block);
+ if (ex instanceof IOException) {
+ flusher.getTargetMetrics().incNumWriteIOExceptionRetryBlocks();
+ } else {
+ flusher.getTargetMetrics().incNumWriteGenericExceptionRetryBlocks();
+ }
+ if (this.getTryCount() >= maxRetryCount) {
+ flusher.getTargetMetrics().incNumWriteMaxRetryBlocks();
+ }
+ } finally {
+ flusher.incFinishCount(fileName);
+ if (levelDBStore != null) {
+ flusher.releaseCacheDB(dbPath);
+ }
+ if(client != null) {
+ flusher.getXceiverClientManager().releaseClient(client);
+ }
+ }
+ }
+
+
+ private void writeRetryBlock(LogicalBlock currentBlock) {
+ boolean append = false;
+ String retryFileName =
+ String.format("%s.%d.%s.%s", AsyncBlockWriter.RETRY_LOG_PREFIX,
+ currentBlock.getBlockID(), Time.monotonicNow(), tryCount);
+ File logDir = new File(this.dbPath);
+ if (!logDir.exists() && !logDir.mkdirs()) {
+ flusher.getLOG().error(
+ "Unable to create the log directory, Critical error cannot continue");
+ return;
+ }
+ String log = Paths.get(this.dbPath, retryFileName).toString();
+ ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
+ buffer.putLong(currentBlock.getBlockID());
+ buffer.flip();
+ try {
+ FileChannel channel = new FileOutputStream(log, append).getChannel();
+ channel.write(buffer);
+ channel.close();
+ flusher.processDirtyBlocks(this.dbPath, retryFileName);
+ } catch (IOException e) {
+ flusher.getTargetMetrics().incNumFailedRetryLogFileWrites();
+ flusher.getLOG().error("Unable to write the retry block. Block ID: {}",
+ currentBlock.getBlockID(), e);
+ }
+ }
+
+ /**
+ * Increments the try count. This is done each time we try this block
+ * write to the container.
+ */
+ private void incTryCount() {
+ tryCount++;
+ }
+
+ /**
+ * Get the retry count.
+ *
+ * @return int
+ */
+ public int getTryCount() {
+ return tryCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
new file mode 100644
index 0000000..84b68e3
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.primitives.Longs;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos.ContainerIDProto;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos.ListVolumesRequestProto;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos.ListVolumesResponseProto;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos.MountVolumeRequestProto;
+import org.apache.hadoop.cblock.protocol.proto
+ .CBlockClientServerProtocolProtos.MountVolumeResponseProto;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos
+ .VolumeInfoProto;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.ipc.ProtobufHelper;
+import org.apache.hadoop.ipc.ProtocolTranslator;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * The client side of CBlockClientProtocol.
+ *
+ * CBlockClientProtocol is the protocol used between cblock client side
+ * and cblock manager (cblock client side is just the node where jscsi daemon
+ * process runs. a machines talks to jscsi daemon for mounting a volume).
+ *
+ * Right now, the only communication carried by this protocol is for client side
+ * to request mounting a volume.
+ */
+public class CBlockClientProtocolClientSideTranslatorPB
+ implements CBlockClientProtocol, ProtocolTranslator, Closeable {
+
+ private final CBlockClientServerProtocolPB rpcProxy;
+
+ public CBlockClientProtocolClientSideTranslatorPB(
+ CBlockClientServerProtocolPB rpcProxy) {
+ this.rpcProxy = rpcProxy;
+ }
+
+
+ @Override
+ public void close() throws IOException {
+ RPC.stopProxy(rpcProxy);
+ }
+
+ @Override
+ public Object getUnderlyingProxyObject() {
+ return rpcProxy;
+ }
+
+ @Override
+ public MountVolumeResponse mountVolume(
+ String userName, String volumeName) throws IOException {
+ MountVolumeRequestProto.Builder
+ request
+ = MountVolumeRequestProto
+ .newBuilder();
+ request.setUserName(userName);
+ request.setVolumeName(volumeName);
+ try {
+ MountVolumeResponseProto resp
+ = rpcProxy.mountVolume(null, request.build());
+ if (!resp.getIsValid()) {
+ throw new CBlockException(
+ "Not a valid volume:" + userName + ":" + volumeName);
+ }
+ List<Pipeline> containerIDs = new ArrayList<>();
+ HashMap<String, Pipeline> containerPipelines = new HashMap<>();
+ if (resp.getAllContainerIDsList().size() == 0) {
+ throw new CBlockException("Mount volume request returned no container");
+ }
+ for (ContainerIDProto containerID :
+ resp.getAllContainerIDsList()) {
+ if (containerID.hasPipeline()) {
+ // it should always have a pipeline only except for tests.
+ Pipeline p = Pipeline.getFromProtoBuf(containerID.getPipeline());
+ p.setData(Longs.toByteArray(containerID.getIndex()));
+ containerIDs.add(p);
+ containerPipelines.put(containerID.getContainerID(), p);
+ } else {
+ throw new CBlockException("ContainerID does not have pipeline!");
+ }
+ }
+ return new MountVolumeResponse(
+ resp.getIsValid(),
+ resp.getUserName(),
+ resp.getVolumeName(),
+ resp.getVolumeSize(),
+ resp.getBlockSize(),
+ containerIDs,
+ containerPipelines);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public List<VolumeInfo> listVolumes() throws IOException {
+ try {
+ List<VolumeInfo> result = new ArrayList<>();
+ ListVolumesResponseProto
+ listVolumesResponseProto = this.rpcProxy.listVolumes(null,
+ ListVolumesRequestProto.newBuilder()
+ .build());
+ for (VolumeInfoProto volumeInfoProto :
+ listVolumesResponseProto
+ .getVolumeEntryList()) {
+ result.add(new VolumeInfo(volumeInfoProto.getUserName(),
+ volumeInfoProto.getVolumeName(), volumeInfoProto.getVolumeSize(),
+ volumeInfoProto.getBlockSize()));
+ }
+ return result;
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
new file mode 100644
index 0000000..2f35668
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
@@ -0,0 +1,440 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.jscsi.target.storage.IStorageModule;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_TRACE_IO_DEFAULT;
+
+/**
+ * The SCSI Target class for CBlockSCSIServer.
+ */
+final public class CBlockIStorageImpl implements IStorageModule {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(CBlockIStorageImpl.class);
+ private static final Logger TRACER =
+ LoggerFactory.getLogger("TraceIO");
+
+ private CacheModule cache;
+ private final long volumeSize;
+ private final int blockSize;
+ private final String userName;
+ private final String volumeName;
+ private final boolean traceEnabled;
+ private final Configuration conf;
+ private final ContainerCacheFlusher flusher;
+ private List<Pipeline> fullContainerList;
+
+ /**
+ * private: constructs a SCSI Target.
+ *
+ * @param config - config
+ * @param userName - Username
+ * @param volumeName - Name of the volume
+ * @param volumeSize - Size of the volume
+ * @param blockSize - Size of the block
+ * @param fullContainerList - Ordered list of containers that make up this
+ * volume.
+ * @param flusher - flusher which is used to flush data from
+ * level db cache to containers
+ * @throws IOException - Throws IOException.
+ */
+ private CBlockIStorageImpl(Configuration config, String userName,
+ String volumeName, long volumeSize, int blockSize,
+ List<Pipeline> fullContainerList, ContainerCacheFlusher flusher) {
+ this.conf = config;
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.volumeSize = volumeSize;
+ this.blockSize = blockSize;
+ this.fullContainerList = new ArrayList<>(fullContainerList);
+ this.flusher = flusher;
+ this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
+ DFS_CBLOCK_TRACE_IO_DEFAULT);
+ }
+
+ /**
+ * private: initialize the cache.
+ *
+ * @param xceiverClientManager - client manager that is used for creating new
+ * connections to containers.
+ * @param metrics - target metrics to maintain metrics for target server
+ * @throws IOException - Throws IOException.
+ */
+ private void initCache(XceiverClientManager xceiverClientManager,
+ CBlockTargetMetrics metrics) throws IOException {
+ this.cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(conf)
+ .setVolumeName(this.volumeName)
+ .setUserName(this.userName)
+ .setPipelines(this.fullContainerList)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(blockSize)
+ .setVolumeSize(volumeSize)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ this.cache.start();
+ }
+
+ /**
+ * Gets a new builder for CBlockStorageImpl.
+ *
+ * @return builder
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ /**
+ * Get Cache.
+ *
+ * @return - Cache
+ */
+ public CacheModule getCache() {
+ return cache;
+ }
+
+ /**
+ * Returns block size of this volume.
+ *
+ * @return int size of block for this volume.
+ */
+ @Override
+ public int getBlockSize() {
+ return blockSize;
+ }
+
+ /**
+ * Checks the index boundary of a block address.
+ *
+ * @param logicalBlockAddress the index of the first block of data to be read
+ * or written
+ * @param transferLengthInBlocks the total number of consecutive blocks about
+ * to be read or written
+ * @return 0 == Success, 1 indicates the LBA address is out of bounds and 2
+ * indicates that LBA + transfer size is out of bounds.
+ */
+ @Override
+ public int checkBounds(long logicalBlockAddress, int transferLengthInBlocks) {
+ long sizeInBlocks = volumeSize / blockSize;
+ int res = 0;
+ if (logicalBlockAddress < 0 || logicalBlockAddress >= sizeInBlocks) {
+ res = 1;
+ }
+
+ if (transferLengthInBlocks < 0 ||
+ logicalBlockAddress + transferLengthInBlocks > sizeInBlocks) {
+ if (res == 0) {
+ res = 2;
+ }
+ }
+ return res;
+ }
+
+ /**
+ * Number of blocks that make up this volume.
+ *
+ * @return long - count of blocks.
+ */
+ @Override
+ public long getSizeInBlocks() {
+ return volumeSize / blockSize;
+ }
+
+ /**
+ * Reads the number of bytes that can be read into the bytes buffer from the
+ * location indicated.
+ *
+ * @param bytes the array into which the data will be copied will be filled
+ * with data from storage
+ * @param storageIndex the position of the first byte to be copied
+ * @throws IOException
+ */
+ @Override
+ public void read(byte[] bytes, long storageIndex) throws IOException {
+ int startingIdxInBlock = (int) storageIndex % blockSize;
+ int idxInBytes = 0;
+ if (this.traceEnabled) {
+ TRACER.info("Task=ReadStart,length={},location={}",
+ bytes.length, storageIndex);
+ }
+ while (idxInBytes < bytes.length - 1) {
+ long blockId = (storageIndex + idxInBytes) / blockSize;
+ byte[] dataBytes;
+
+ try {
+ LogicalBlock block = this.cache.get(blockId);
+ dataBytes = block.getData().array();
+
+ if (this.traceEnabled) {
+ TRACER.info("Task=ReadBlock,BlockID={},length={},SHA={}",
+ blockId,
+ dataBytes.length,
+ dataBytes.length > 0 ? DigestUtils.sha256Hex(dataBytes) : null);
+ }
+ } catch (IOException e) {
+ // For an non-existing block cache.get will return a block with zero
+ // bytes filled. So any error here is a real error.
+ LOGGER.error("getting errors when reading data:" + e);
+ throw e;
+ }
+
+ int length = blockSize - startingIdxInBlock;
+ if (length > bytes.length - idxInBytes) {
+ length = bytes.length - idxInBytes;
+ }
+ if (dataBytes.length >= length) {
+ System.arraycopy(dataBytes, startingIdxInBlock, bytes, idxInBytes,
+ length);
+ }
+ startingIdxInBlock = 0;
+ idxInBytes += length;
+ }
+ if (this.traceEnabled) {
+ TRACER.info("Task=ReadEnd,length={},location={},SHA={}",
+ bytes.length, storageIndex, DigestUtils.sha256Hex(bytes));
+ }
+ }
+
+ @Override
+ public void write(byte[] bytes, long storageIndex) throws IOException {
+ int startingIdxInBlock = (int) storageIndex % blockSize;
+ int idxInBytes = 0;
+ if (this.traceEnabled) {
+ TRACER.info("Task=WriteStart,length={},location={},SHA={}",
+ bytes.length, storageIndex,
+ bytes.length > 0 ? DigestUtils.sha256Hex(bytes) : null);
+ }
+
+ ByteBuffer dataByte = ByteBuffer.allocate(blockSize);
+ while (idxInBytes < bytes.length - 1) {
+ long blockId = (storageIndex + idxInBytes) / blockSize;
+ int length = blockSize - startingIdxInBlock;
+ if (length > bytes.length - idxInBytes) {
+ length = bytes.length - idxInBytes;
+ }
+ System.arraycopy(bytes, idxInBytes, dataByte.array(), startingIdxInBlock,
+ length);
+ this.cache.put(blockId, dataByte.array());
+
+ if (this.traceEnabled) {
+ TRACER.info("Task=WriteBlock,BlockID={},length={},SHA={}",
+ blockId, dataByte.array().length,
+ dataByte.array().length > 0 ?
+ DigestUtils.sha256Hex(dataByte.array()) : null);
+ }
+ dataByte.clear();
+ startingIdxInBlock = 0;
+ idxInBytes += length;
+ }
+
+ if (this.traceEnabled) {
+ TRACER.info("Task=WriteEnd,length={},location={} ",
+ bytes.length, storageIndex);
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ cache.close();
+ } catch (IllegalStateException ise) {
+ LOGGER.error("Can not close the storage {}", ise);
+ throw ise;
+ }
+ }
+
+ /**
+ * Builder class for CBlocklocalCache.
+ */
+ public static class Builder {
+ private String userName;
+ private String volumeName;
+ private long volumeSize;
+ private int blockSize;
+ private List<Pipeline> containerList;
+ private Configuration conf;
+ private XceiverClientManager clientManager;
+ private ContainerCacheFlusher flusher;
+ private CBlockTargetMetrics metrics;
+
+ /**
+ * Constructs a builder.
+ */
+ Builder() {
+
+ }
+
+ public Builder setFlusher(ContainerCacheFlusher cacheFlusher) {
+ this.flusher = cacheFlusher;
+ return this;
+ }
+
+ /**
+ * set config.
+ *
+ * @param config - config
+ * @return Builder
+ */
+ public Builder setConf(Configuration config) {
+ this.conf = config;
+ return this;
+ }
+
+ /**
+ * set user name.
+ *
+ * @param cblockUserName - user name
+ * @return Builder
+ */
+ public Builder setUserName(String cblockUserName) {
+ this.userName = cblockUserName;
+ return this;
+ }
+
+ /**
+ * set volume name.
+ *
+ * @param cblockVolumeName -- volume name
+ * @return Builder
+ */
+ public Builder setVolumeName(String cblockVolumeName) {
+ this.volumeName = cblockVolumeName;
+ return this;
+ }
+
+ /**
+ * set volume size.
+ *
+ * @param cblockVolumeSize -- set volume size.
+ * @return Builder
+ */
+ public Builder setVolumeSize(long cblockVolumeSize) {
+ this.volumeSize = cblockVolumeSize;
+ return this;
+ }
+
+ /**
+ * set block size.
+ *
+ * @param cblockBlockSize -- block size
+ * @return Builder
+ */
+ public Builder setBlockSize(int cblockBlockSize) {
+ this.blockSize = cblockBlockSize;
+ return this;
+ }
+
+ /**
+ * Set contianer list.
+ *
+ * @param cblockContainerList - set the pipeline list
+ * @return Builder
+ */
+ public Builder setContainerList(List<Pipeline> cblockContainerList) {
+ this.containerList = cblockContainerList;
+ return this;
+ }
+
+ /**
+ * Set client manager.
+ *
+ * @param xceiverClientManager -- sets the client manager.
+ * @return Builder
+ */
+ public Builder setClientManager(XceiverClientManager xceiverClientManager) {
+ this.clientManager = xceiverClientManager;
+ return this;
+ }
+
+ /**
+ * Set Cblock Target Metrics.
+ *
+ * @param targetMetrics -- sets the cblock target metrics
+ * @return Builder
+ */
+ public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+ this.metrics = targetMetrics;
+ return this;
+ }
+
+ /**
+ * Builds the CBlockStorageImpl.
+ *
+ * @return builds the CBlock Scsi Target.
+ */
+ public CBlockIStorageImpl build() throws IOException {
+ if (StringUtils.isBlank(userName)) {
+ throw new IllegalArgumentException("User name cannot be null or empty" +
+ ".");
+ }
+ if (StringUtils.isBlank(volumeName)) {
+ throw new IllegalArgumentException("Volume name cannot be null or " +
+ "empty");
+ }
+
+ if (volumeSize < 1) {
+ throw new IllegalArgumentException("Volume size cannot be negative or" +
+ " zero.");
+ }
+
+ if (blockSize < 1) {
+ throw new IllegalArgumentException("Block size cannot be negative or " +
+ "zero.");
+ }
+
+ if (containerList == null || containerList.size() == 0) {
+ throw new IllegalArgumentException("Container list cannot be null or " +
+ "empty");
+ }
+ if (clientManager == null) {
+ throw new IllegalArgumentException("Client manager cannot be null");
+ }
+ if (conf == null) {
+ throw new IllegalArgumentException("Configuration cannot be null");
+ }
+
+ if (flusher == null) {
+ throw new IllegalArgumentException("Flusher Cannot be null.");
+ }
+ CBlockIStorageImpl impl = new CBlockIStorageImpl(this.conf, this.userName,
+ this.volumeName, this.volumeSize, this.blockSize, this.containerList,
+ this.flusher);
+ impl.initCache(this.clientManager, this.metrics);
+ return impl;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
new file mode 100644
index 0000000..6367c61
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * This class is the handler of CBlockManager used by target server
+ * to communicate with CBlockManager.
+ *
+ * More specifically, this class will expose local methods to target
+ * server, and make RPC calls to CBlockManager accordingly
+ */
+public class CBlockManagerHandler {
+
+ private final CBlockClientProtocolClientSideTranslatorPB handler;
+
+ public CBlockManagerHandler(
+ CBlockClientProtocolClientSideTranslatorPB handler) {
+ this.handler = handler;
+ }
+
+ public MountVolumeResponse mountVolume(
+ String userName, String volumeName) throws IOException {
+ return handler.mountVolume(userName, volumeName);
+ }
+
+ public List<VolumeInfo> listVolumes() throws IOException {
+ return handler.listVolumes();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
new file mode 100644
index 0000000..b9ec462
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.meta.ContainerDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeDescriptor;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos;
+import org.apache.hadoop.scm.client.ScmClient;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * This class maintains the key space of CBlock, more specifically, the
+ * volume to container mapping. The core data structure
+ * is a map from users to their volumes info, where volume info is a handler
+ * to a volume, containing information for IO on that volume and a storage
+ * client responsible for talking to the SCM.
+ */
+public class StorageManager {
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(StorageManager.class);
+ private final ScmClient storageClient;
+ private final int numThreads;
+ private static final int MAX_THREADS =
+ Runtime.getRuntime().availableProcessors() * 2;
+ private static final int MAX_QUEUE_CAPACITY = 1024;
+ private final String cblockId;
+
+ /**
+ * We will NOT have the situation where same kv pair getting
+ * processed, but it is possible to have multiple kv pair being
+ * processed at same time.
+ *
+ * So using just ConcurrentHashMap should be sufficient
+ *
+ * Again since currently same user accessing from multiple places
+ * is not allowed, no need to consider concurrency of volume map
+ * within one user
+ */
+ private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>>
+ user2VolumeMap;
+ // size of an underlying container.
+ // TODO : assuming all containers are of the same size
+ private long containerSizeB;
+
+ public StorageManager(ScmClient storageClient,
+ OzoneConfiguration ozoneConfig, String cblockId) throws IOException {
+ this.storageClient = storageClient;
+ this.user2VolumeMap = new ConcurrentHashMap<>();
+ this.containerSizeB = storageClient.getContainerSize(null);
+ this.numThreads =
+ ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE,
+ CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT);
+ this.cblockId = cblockId;
+ }
+
+ /**
+ * This call will put the volume into in-memory map.
+ *
+ * more specifically, make the volume discoverable on jSCSI server
+ * and keep it's reference in-memory for look up.
+ * @param userName the user name of the volume.
+ * @param volumeName the name of the volume,
+ * @param volume a {@link VolumeDescriptor} object encapsulating the
+ * information about the volume.
+ */
+ private void makeVolumeReady(String userName, String volumeName,
+ VolumeDescriptor volume) {
+ HashMap<String, VolumeDescriptor> userVolumes;
+ if (user2VolumeMap.containsKey(userName)) {
+ userVolumes = user2VolumeMap.get(userName);
+ } else {
+ userVolumes = new HashMap<>();
+ user2VolumeMap.put(userName, userVolumes);
+ }
+ userVolumes.put(volumeName, volume);
+ }
+
+ /**
+ * Called by CBlockManager to add volumes read from persistent store into
+ * memory, need to contact SCM to setup the reference to the containers given
+ * their id.
+ *
+ * Only for failover process where container meta info is read from
+ * persistent store, and containers themselves are alive.
+ *
+ * TODO : Currently, this method is not being called as failover process
+ * is not implemented yet.
+ *
+ * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating
+ * the information about a volume.
+ * @throws IOException when adding the volume failed. e.g. volume already
+ * exist, or no more container available.
+ */
+ public synchronized void addVolume(VolumeDescriptor volumeDescriptor)
+ throws IOException{
+ String userName = volumeDescriptor.getUserName();
+ String volumeName = volumeDescriptor.getVolumeName();
+ LOGGER.info("addVolume:" + userName + ":" + volumeName);
+ if (user2VolumeMap.containsKey(userName)
+ && user2VolumeMap.get(userName).containsKey(volumeName)) {
+ throw new CBlockException("Volume already exist for "
+ + userName + ":" + volumeName);
+ }
+ // the container ids are read from levelDB, setting up the
+ // container handlers here.
+ String[] containerIds = volumeDescriptor.getContainerIDs();
+
+ for (String containerId : containerIds) {
+ try {
+ Pipeline pipeline = storageClient.getContainer(containerId);
+ ContainerDescriptor containerDescriptor =
+ new ContainerDescriptor(containerId);
+ containerDescriptor.setPipeline(pipeline);
+ volumeDescriptor.addContainer(containerDescriptor);
+ } catch (IOException e) {
+ LOGGER.error("Getting container failed! Container:{} error:{}",
+ containerId, e);
+ throw e;
+ }
+ }
+ // now ready to put into in-memory map.
+ makeVolumeReady(userName, volumeName, volumeDescriptor);
+ }
+
+ private class CreateContainerTask implements Runnable {
+ private final VolumeDescriptor volume;
+ private final int containerIdx;
+ private final ArrayList<String> containerIds;
+ private final AtomicInteger numFailed;
+
+ CreateContainerTask(VolumeDescriptor volume, int containerIdx,
+ ArrayList<String> containerIds,
+ AtomicInteger numFailed) {
+ this.volume = volume;
+ this.containerIdx = containerIdx;
+ this.containerIds = containerIds;
+ this.numFailed = numFailed;
+ }
+
+ /**
+ * When an object implementing interface <code>Runnable</code> is used
+ * to create a thread, starting the thread causes the object's
+ * <code>run</code> method to be called in that separately executing
+ * thread.
+ * <p>
+ * The general contract of the method <code>run</code> is that it may
+ * take any action whatsoever.
+ *
+ * @see Thread#run()
+ */
+ public void run() {
+ ContainerDescriptor container = null;
+ try {
+ Pipeline pipeline = storageClient.createContainer(
+ HdslProtos.ReplicationType.STAND_ALONE,
+ HdslProtos.ReplicationFactor.ONE,
+ KeyUtil.getContainerName(volume.getUserName(),
+ volume.getVolumeName(), containerIdx), cblockId);
+
+ container = new ContainerDescriptor(pipeline.getContainerName());
+
+ container.setPipeline(pipeline);
+ container.setContainerIndex(containerIdx);
+ volume.addContainer(container);
+ containerIds.set(containerIdx, container.getContainerID());
+ } catch (Exception e) {
+ numFailed.incrementAndGet();
+ if (container != null) {
+ LOGGER.error("Error creating container Container:{}:" +
+ " index:{} error:{}", container.getContainerID(),
+ containerIdx, e);
+ } else {
+ LOGGER.error("Error creating container.", e);
+ }
+ }
+ }
+ }
+
+ private boolean createVolumeContainers(VolumeDescriptor volume) {
+ ArrayList<String> containerIds = new ArrayList<>();
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(
+ Math.min(numThreads, MAX_THREADS),
+ MAX_THREADS, 1, TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+
+ AtomicInteger numFailedCreates = new AtomicInteger(0);
+ long allocatedSize = 0;
+ int containerIdx = 0;
+ while (allocatedSize < volume.getVolumeSize()) {
+ // adding null to allocate space in ArrayList
+ containerIds.add(containerIdx, null);
+ Runnable task = new CreateContainerTask(volume, containerIdx,
+ containerIds, numFailedCreates);
+ executor.submit(task);
+ allocatedSize += containerSizeB;
+ containerIdx += 1;
+ }
+
+ // issue the command and then wait for it to finish
+ executor.shutdown();
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ LOGGER.error("Error creating volume:{} error:{}",
+ volume.getVolumeName(), e);
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+
+ volume.setContainerIDs(containerIds);
+ return numFailedCreates.get() == 0;
+ }
+
+ private void deleteContainer(String containerID, boolean force) {
+ try {
+ Pipeline pipeline = storageClient.getContainer(containerID);
+ storageClient.deleteContainer(pipeline, force);
+ } catch (Exception e) {
+ LOGGER.error("Error deleting container Container:{} error:{}",
+ containerID, e);
+ }
+ }
+
+ private void deleteVolumeContainers(List<String> containers, boolean force)
+ throws CBlockException {
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(
+ Math.min(numThreads, MAX_THREADS),
+ MAX_THREADS, 1, TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
+ new ThreadPoolExecutor.CallerRunsPolicy());
+
+ for (String deleteContainer : containers) {
+ if (deleteContainer != null) {
+ Runnable task = () -> deleteContainer(deleteContainer, force);
+ executor.submit(task);
+ }
+ }
+
+ // issue the command and then wait for it to finish
+ executor.shutdown();
+ try {
+ executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ LOGGER.error("Error deleting containers error:{}", e);
+ executor.shutdownNow();
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ /**
+ * Called by CBlock server when creating a fresh volume. The core
+ * logic is adding needed information into in-memory meta data.
+ *
+ * @param userName the user name of the volume.
+ * @param volumeName the name of the volume.
+ * @param volumeSize the size of the volume.
+ * @param blockSize the block size of the volume.
+ * @throws CBlockException when the volume can not be created.
+ */
+ public synchronized void createVolume(String userName, String volumeName,
+ long volumeSize, int blockSize) throws CBlockException {
+ LOGGER.debug("createVolume:" + userName + ":" + volumeName);
+ if (user2VolumeMap.containsKey(userName)
+ && user2VolumeMap.get(userName).containsKey(volumeName)) {
+ throw new CBlockException("Volume already exist for "
+ + userName + ":" + volumeName);
+ }
+ if (volumeSize < blockSize) {
+ throw new CBlockException("Volume size smaller than block size? " +
+ "volume size:" + volumeSize + " block size:" + blockSize);
+ }
+ VolumeDescriptor volume
+ = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize);
+ boolean success = createVolumeContainers(volume);
+ if (!success) {
+ // cleanup the containers and throw the exception
+ deleteVolumeContainers(volume.getContainerIDsList(), true);
+ throw new CBlockException("Error when creating volume:" + volumeName);
+ }
+ makeVolumeReady(userName, volumeName, volume);
+ }
+
+ /**
+ * Called by CBlock server to delete a specific volume. Mainly
+ * to check whether it can be deleted, and remove it from in-memory meta
+ * data.
+ *
+ * @param userName the user name of the volume.
+ * @param volumeName the name of the volume.
+ * @param force if set to false, only delete volume it is empty, otherwise
+ * throw exception. if set to true, delete regardless.
+ * @throws CBlockException when the volume can not be deleted.
+ */
+ public synchronized void deleteVolume(String userName, String volumeName,
+ boolean force) throws CBlockException {
+ if (!user2VolumeMap.containsKey(userName)
+ || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+ throw new CBlockException("Deleting non-exist volume "
+ + userName + ":" + volumeName);
+ }
+ if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) {
+ throw new CBlockException("Deleting a non-empty volume without force!");
+ }
+ VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName);
+ deleteVolumeContainers(volume.getContainerIDsList(), force);
+ if (user2VolumeMap.get(userName).size() == 0) {
+ user2VolumeMap.remove(userName);
+ }
+ }
+
+ /**
+ * Called by CBlock server to get information of a specific volume.
+ *
+ * @param userName the user name of the volume.
+ * @param volumeName the name of the volume.
+ * @return a {@link VolumeInfo} object encapsulating the information of the
+ * volume.
+ * @throws CBlockException when the information can not be retrieved.
+ */
+ public synchronized VolumeInfo infoVolume(String userName, String volumeName)
+ throws CBlockException {
+ if (!user2VolumeMap.containsKey(userName)
+ || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+ throw new CBlockException("Getting info for non-exist volume "
+ + userName + ":" + volumeName);
+ }
+ return user2VolumeMap.get(userName).get(volumeName).getInfo();
+ }
+
+ /**
+ * Called by CBlock server to check whether the given volume can be
+ * mounted, i.e. whether it can be found in the meta data.
+ *
+ * return a {@link MountVolumeResponse} with isValid flag to indicate
+ * whether the volume can be mounted or not.
+ *
+ * @param userName the user name of the volume.
+ * @param volumeName the name of the volume
+ * @return a {@link MountVolumeResponse} object encapsulating whether the
+ * volume is valid, and if yes, the requried information for client to
+ * read/write the volume.
+ */
+ public synchronized MountVolumeResponse isVolumeValid(
+ String userName, String volumeName) {
+ if (!user2VolumeMap.containsKey(userName)
+ || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+ // in the case of invalid volume, no need to set any value other than
+ // isValid flag.
+ return new MountVolumeResponse(false, null, null, 0, 0, null, null);
+ }
+ VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName);
+ return new MountVolumeResponse(true, userName,
+ volumeName, volume.getVolumeSize(), volume.getBlockSize(),
+ volume.getContainerPipelines(), volume.getPipelines());
+ }
+
+ /**
+ * Called by CBlock manager to list all volumes.
+ *
+ * @param userName the userName whose volume to be listed, if set to null,
+ * all volumes will be listed.
+ * @return a list of {@link VolumeDescriptor} representing all volumes
+ * requested.
+ */
+ public synchronized List<VolumeDescriptor> getAllVolume(String userName) {
+ ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>();
+ if (userName == null) {
+ for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry
+ : user2VolumeMap.entrySet()) {
+ allVolumes.addAll(entry.getValue().values());
+ }
+ } else {
+ if (user2VolumeMap.containsKey(userName)) {
+ allVolumes.addAll(user2VolumeMap.get(userName).values());
+ }
+ }
+ return allVolumes;
+ }
+
+ /**
+ * Only for testing the behavior of create/delete volumes.
+ */
+ @VisibleForTesting
+ public VolumeDescriptor getVolume(String userName, String volumeName) {
+ if (!user2VolumeMap.containsKey(userName)
+ || !user2VolumeMap.get(userName).containsKey(volumeName)) {
+ return null;
+ }
+ return user2VolumeMap.get(userName).get(volumeName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
new file mode 100644
index 0000000..4426e6d
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.storage;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
new file mode 100644
index 0000000..beb9e32
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
+
+/**
+ * A simply class that generates key mappings. (e.g. from (userName, volumeName)
+ * pair to a single string volumeKey.
+ */
+public final class KeyUtil {
+ public static String getVolumeKey(String userName, String volumeName) {
+ return userName + ":" + volumeName;
+ }
+
+ public static String getContainerName(String userName, String volumeName,
+ int containerID) {
+ return getVolumeKey(userName, volumeName) + "#" + containerID;
+ }
+
+ public static String getUserNameFromVolumeKey(String key) {
+ return key.split(":")[0];
+ }
+
+ public static String getVolumeFromVolumeKey(String key) {
+ return key.split(":")[1];
+ }
+
+ public static boolean isValidVolumeKey(String key) {
+ return key.contains(":");
+ }
+
+ private KeyUtil() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
new file mode 100644
index 0000000..5b9aa0c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/util/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.util;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
new file mode 100644
index 0000000..160b254
--- /dev/null
+++ b/hadoop-cblock/server/src/main/proto/CBlockClientServerProtocol.proto
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+option java_package = "org.apache.hadoop.cblock.protocol.proto";
+option java_outer_classname = "CBlockClientServerProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.cblock;
+
+import "hdsl.proto";
+import "CBlockServiceProtocol.proto";
+/**
+* This message is sent from CBlock client side to CBlock server to
+* mount a volume specified by owner name and volume name.
+*
+* Right now, this is the only communication between client and server.
+* After the volume is mounted, CBlock client will talk to containers
+* by itself, nothing to do with CBlock server.
+*/
+message MountVolumeRequestProto {
+ required string userName = 1;
+ required string volumeName = 2;
+}
+
+/**
+* This message is sent from CBlock server to CBlock client as response
+* of mount a volume. It checks the whether the volume is valid to access
+* at all.(e.g. volume exist)
+*
+* And include enough information (volume size, block size, list of
+* containers for this volume) for client side to perform read/write on
+* the volume.
+*/
+message MountVolumeResponseProto {
+ required bool isValid = 1;
+ optional string userName = 2;
+ optional string volumeName = 3;
+ optional uint64 volumeSize = 4;
+ optional uint32 blockSize = 5;
+ repeated ContainerIDProto allContainerIDs = 6;
+}
+
+/**
+* This message include ID of container which can be used to locate the
+* container. Since the order of containers needs to be maintained, also
+* includes a index field to verify the correctness of the order.
+*/
+message ContainerIDProto {
+ required string containerID = 1;
+ required uint64 index = 2;
+ // making pipeline optional to be compatible with exisiting tests
+ optional hadoop.hdsl.Pipeline pipeline = 3;
+}
+
+
+message ListVolumesRequestProto {
+
+}
+
+message ListVolumesResponseProto {
+ repeated VolumeInfoProto volumeEntry = 1;
+}
+
+
+service CBlockClientServerProtocolService {
+ /**
+ * mount the volume.
+ */
+ rpc mountVolume(MountVolumeRequestProto) returns (MountVolumeResponseProto);
+
+ rpc listVolumes(ListVolumesRequestProto) returns(ListVolumesResponseProto);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto b/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
new file mode 100644
index 0000000..36e4b59
--- /dev/null
+++ b/hadoop-cblock/server/src/main/proto/CBlockServiceProtocol.proto
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * These .proto interfaces are private and unstable.
+ * Please see http://wiki.apache.org/hadoop/Compatibility
+ * for what changes are allowed for a *unstable* .proto interface.
+ */
+
+option java_package = "org.apache.hadoop.cblock.protocol.proto";
+option java_outer_classname = "CBlockServiceProtocolProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.cblock;
+
+/**
+* This message is sent to CBlock server to create a volume. Creating
+* volume requries four parameters: owner of the volume, name of the volume
+* size of volume and block size of the volume.
+*/
+message CreateVolumeRequestProto {
+ required string userName = 1;
+ required string volumeName = 2;
+ required uint64 volumeSize = 3;
+ optional uint32 blockSize = 4 [default = 4096];
+}
+
+/**
+* Empty response message.
+*/
+message CreateVolumeResponseProto {
+
+}
+
+/**
+* This message is sent to CBlock server to delete a volume. The volume
+* is specified by owner name and volume name. If force is set to
+* false, volume will be deleted only if it is empty. Otherwise delete it
+* regardless.
+*/
+message DeleteVolumeRequestProto {
+ required string userName = 1;
+ required string volumeName = 2;
+ optional bool force = 3;
+}
+
+/**
+* Empty response message.
+*/
+message DeleteVolumeResponseProto {
+
+}
+
+/**
+* This message is sent to CBlock server to request info of a volume. The
+* volume is specified by owner name and volume name.
+*/
+message InfoVolumeRequestProto {
+ required string userName = 1;
+ required string volumeName = 2;
+}
+
+/**
+* This message describes the information of a volume.
+* Currently, the info includes the volume creation parameters and a number
+* as the usage of the volume, in terms of number of bytes.
+*/
+message VolumeInfoProto {
+ required string userName = 1;
+ required string volumeName = 2;
+ required uint64 volumeSize = 3;
+ required uint64 blockSize = 4;
+ optional uint64 usage = 5;
+ // TODO : potentially volume ACL
+}
+
+/**
+* This message is sent from CBlock server as response of info volume request.
+*/
+message InfoVolumeResponseProto {
+ optional VolumeInfoProto volumeInfo = 1;
+}
+
+/**
+* This message is sent to CBlock server to list all available volume.
+*/
+message ListVolumeRequestProto {
+ optional string userName = 1;
+}
+
+/**
+* This message is sent from CBlock server as response of volume listing.
+*/
+message ListVolumeResponseProto {
+ repeated VolumeInfoProto volumeEntry = 1;
+}
+
+service CBlockServiceProtocolService {
+ /**
+ * Create a volume.
+ */
+ rpc createVolume(CreateVolumeRequestProto) returns(CreateVolumeResponseProto);
+
+ /**
+ * Delete a volume.
+ */
+ rpc deleteVolume(DeleteVolumeRequestProto) returns(DeleteVolumeResponseProto);
+
+ /**
+ * Get info of a volume.
+ */
+ rpc infoVolume(InfoVolumeRequestProto) returns(InfoVolumeResponseProto);
+
+ /**
+ * List all available volumes.
+ */
+ rpc listVolume(ListVolumeRequestProto) returns(ListVolumeResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/resources/cblock-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/resources/cblock-default.xml b/hadoop-cblock/server/src/main/resources/cblock-default.xml
new file mode 100644
index 0000000..ebf36cd
--- /dev/null
+++ b/hadoop-cblock/server/src/main/resources/cblock-default.xml
@@ -0,0 +1,347 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<!-- Do not modify this file directly. Instead, copy entries that you -->
+<!-- wish to modify from this file into ozone-site.xml and change them -->
+<!-- there. If ozone-site.xml does not already exist, create it. -->
+
+<!--Tags supported are OZONE, CBLOCK, MANAGEMENT, SECURITY, PERFORMANCE, -->
+<!--DEBUG, CLIENT, SERVER, KSM, SCM, CRITICAL, RATIS, CONTAINER, REQUIRED, -->
+<!--REST, STORAGE, PIPELINE, STANDALONE -->
+
+<configuration>
+
+ <!--CBlock Settings-->
+ <property>
+ <name>dfs.cblock.block.buffer.flush.interval</name>
+ <value>60s</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Controls the frequency at this the local cache flushes the
+ blocks to the remote containers.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.block.buffer.size</name>
+ <value>512</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Size of the local cache for blocks. So cache size will be block
+ size multiplied by this number.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.core.min.pool.size</name>
+ <value>16</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ A minimum number of threads in the pool that cBlock cache will
+ use for the background I/O to remote containers.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.max.pool.size</name>
+ <value>256</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Maximum number of threads in the pool that cBlock cache will
+ use for background I/O to remote containers.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.keep.alive</name>
+ <value>60s</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ If the cblock cache has no I/O, then the threads in the cache
+ pool are kept idle for this amount of time before shutting down.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.leveldb.cache.size.mb</name>
+ <value>256</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ The amount of physical memory allocated to the local cache. The
+ SCSI driver will allocate this much RAM cache instances.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.max.retry</name>
+ <value>65536</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ If the local cache is enabled then, CBlock writes to the local
+ cache when I/O happens. Then the background I/O threads write this
+ block to the remote containers. This value controls how many times the
+ background thread should attempt to do I/O to the remote containers
+ before giving up.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.queue.size.in.kb</name>
+ <value>256</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Size of the in memory cache queue, that is flushed to local
+ disk.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.cache.thread.priority</name>
+ <value>5</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Priority of cache flusher thread, affecting the relative performance of
+ write and read. Supported values are 1, 5, 10.
+ Use 10 for high priority and 1 for low priority.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.container.size.gb</name>
+ <value>5</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The size of ozone container in the number of GBs. Note that
+ this is not setting container size for ozone. This setting is
+ instructing CBlock to manage containers at a standard size.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.disk.cache.path</name>
+ <value>${hadoop.tmp.dir}/cblockCacheDB</value>
+ <tag>CBLOCK, REQUIRED</tag>
+ <description>
+ The default path for the cblock local cache. If the cblock
+ local cache is enabled, then it must be set to a valid path. This cache
+ *should* be mapped to the fastest disk on a given machine, For example,
+ an SSD drive would be a good idea. Currently, all mounted disk on a
+ data node is mapped to a single path, so having a large number of IOPS
+ is essential.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.jscsi-address</name>
+ <value/>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The address that cblock will be bind to, should be a host:port
+ format, This setting is required for cblock server to start.
+ This address to be used by jscsi to mount volume.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.jscsi.cblock.server.address</name>
+ <value>127.0.0.1</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The address local jscsi server will use to talk to cblock manager.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.jscsi.port</name>
+ <value>9811</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The port on CBlockManager node for jSCSI to talk to.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.jscsi.rpc-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The actual address the cblock jscsi rpc server will bind to. If
+ this optional address is set, it overrides only the hostname portion of
+ dfs.cblock.jscsi-address.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.jscsi.server.address</name>
+ <value>0.0.0.0</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The address that jscsi server will be running, it is nice have one
+ local jscsi server for each client(Linux JSCSI client) that tries to
+ mount cblock.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.manager.pool.size</name>
+ <value>16</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Number of active threads that cblock manager will use for container
+ operations. The maximum number of the threads are limited to the
+ processor count * 2.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.rpc.timeout</name>
+ <value>300s</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ RPC timeout used for cblock CLI operations. When you
+ create very large disks, like 5TB, etc. The number of containers
+ allocated in the system is huge. It is will 5TB/5GB, which is 1000
+ containers. The client CLI might timeout even though the cblock manager
+ creates the specified disk. This value allows the user to wait for a
+ longer period.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.scm.ipaddress</name>
+ <value>127.0.0.1</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ IP address used by cblock to connect to SCM.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.scm.port</name>
+ <value>9860</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ Port used by cblock to connect to SCM.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.service.handler.count</name>
+ <value>10</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ Default number of handlers for CBlock service rpc.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.service.leveldb.path</name>
+ <value>${hadoop.tmp.dir}/cblock_server.dat</value>
+ <tag>CBLOCK, REQUIRED</tag>
+ <description>
+ Default path for the cblock meta data store.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.service.rpc-bind-host</name>
+ <value>0.0.0.0</value>
+ <tag>CBLOCK, MANAGEMENT</tag>
+ <description>
+ The actual address the cblock service RPC server will bind to.
+ If the optional address is set, it overrides only the hostname portion of
+ dfs.cblock.servicerpc-address.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.servicerpc-address</name>
+ <value/>
+ <tag>CBLOCK, MANAGEMENT, REQUIRED</tag>
+ <description>
+ The address that cblock will be bind to, should be a host:port
+ format, this setting is required for cblock server to start.
+ This address is used for cblock management operations like create, delete,
+ info and list volumes
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.short.circuit.io</name>
+ <value>false</value>
+ <tag>CBLOCK, PERFORMANCE</tag>
+ <description>
+ Enables use of the local cache in cblock. Enabling this allows
+ I/O against the local cache and background threads do actual I/O against
+ the
+ containers.
+ </description>
+ </property>
+ <property>
+ <name>dfs.cblock.trace.io</name>
+ <value>false</value>
+ <tag>CBLOCK, DEBUG</tag>
+ <description>Default flag for enabling trace io, Trace I/O logs all I/O with
+ hashes of
+ data. This is useful for detecting things like data corruption.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.iscsi.advertised.ip</name>
+ <value>0.0.0.0</value>
+ <tag>CBLOCK</tag>
+ <description>
+ IP address returned during the iscsi discovery.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.iscsi.advertised.port</name>
+ <value>3260</value>
+ <tag>CBLOCK</tag>
+ <description>
+ TCP port returned during the iscsi discovery.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.kubernetes.dynamic-provisioner.enabled</name>
+ <value>false</value>
+ <tag>CBLOCK, KUBERNETES</tag>
+ <description>Flag to enable automatic creation of cblocks and
+ kubernetes PersitentVolumes in kubernetes environment.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.kubernetes.cblock-user</name>
+ <value>iqn.2001-04.org.apache.hadoop</value>
+ <tag>CBLOCK, KUBERNETES</tag>
+ <description>CBlock user to use for the dynamic provisioner.
+ This user will own all of the auto-created cblocks.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.kubernetes.configfile</name>
+ <value></value>
+ <tag>CBLOCK, KUBERNETES</tag>
+ <description>Location of the kubernetes configuration file
+ to access the kubernetes cluster. Not required inside a pod
+ as the default service account will be if this value is
+ empty.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.iscsi.advertised.ip</name>
+ <value></value>
+ <tag>CBLOCK, KUBERNETES</tag>
+ <description>IP where the cblock target server is available
+ from the kubernetes nodes. Usually it's a cluster ip address
+ which is defined by a deployed Service.
+ </description>
+ </property>
+
+ <property>
+ <name>dfs.cblock.iscsi.advertised.port</name>
+ <value>3260</value>
+ <tag>CBLOCK, KUBERNETES</tag>
+ <description>Port where the cblock target server is available
+ from the kubernetes nodes. Could be different from the
+ listening port if jscsi is behind a Service.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
new file mode 100644
index 0000000..e1eb36f
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestBufferManager.java
@@ -0,0 +1,456 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+
+/**
+ * Tests for Local Cache Buffer Manager.
+ */
+public class TestBufferManager {
+ private final static long GB = 1024 * 1024 * 1024;
+ private final static int KB = 1024;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration config;
+ private static StorageContainerLocationProtocolClientSideTranslatorPB
+ storageContainerLocationClient;
+ private static XceiverClientManager xceiverClientManager;
+
+ @BeforeClass
+ public static void init() throws IOException {
+ config = new OzoneConfiguration();
+ String path = GenericTestUtils.getTempPath(
+ TestBufferManager.class.getSimpleName());
+ config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ cluster = new MiniOzoneClassicCluster.Builder(config)
+ .numDataNodes(1).setHandlerType("distributed").build();
+ storageContainerLocationClient = cluster
+ .createStorageContainerLocationClient();
+ xceiverClientManager = new XceiverClientManager(config);
+ }
+
+ @AfterClass
+ public static void shutdown() throws InterruptedException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+ }
+
+ /**
+ * createContainerAndGetPipeline creates a set of containers and returns the
+ * Pipelines that define those containers.
+ *
+ * @param count - Number of containers to create.
+ * @return - List of Pipelines.
+ * @throws IOException
+ */
+ private List<Pipeline> createContainerAndGetPipeline(int count)
+ throws IOException {
+ List<Pipeline> containerPipelines = new LinkedList<>();
+ for (int x = 0; x < count; x++) {
+ String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+ String containerName = "container" + RandomStringUtils.randomNumeric(10);
+ Pipeline pipeline =
+ storageContainerLocationClient.allocateContainer(
+ xceiverClientManager.getType(),
+ xceiverClientManager.getFactor(), containerName, "CBLOCK");
+ XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+ ContainerProtocolCalls.createContainer(client, traceID);
+ // This step is needed since we set private data on pipelines, when we
+ // read the list from CBlockServer. So we mimic that action here.
+ pipeline.setData(Longs.toByteArray(x));
+ containerPipelines.add(pipeline);
+ xceiverClientManager.releaseClient(client);
+ }
+ return containerPipelines;
+ }
+
+ /**
+ * This test writes some block to the cache and then shuts down the cache.
+ * The cache is then restarted to check that the
+ * correct number of blocks are read from Dirty Log
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testEmptyBlockBufferHandling() throws IOException,
+ InterruptedException, TimeoutException {
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ List<Pipeline> pipelines = createContainerAndGetPipeline(10);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ // Write data to the cache
+ cache.put(1, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+ Assert.assertEquals(1, metrics.getNumWriteOps());
+ cache.put(2, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+ Assert.assertEquals(2, metrics.getNumWriteOps());
+
+ // Store the previous block buffer position
+ Assert.assertEquals(2, metrics.getNumBlockBufferUpdates());
+ // Simulate a shutdown by closing the cache
+ cache.close();
+ Thread.sleep(1000);
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+ Assert.assertEquals(2 * (Long.SIZE/ Byte.SIZE),
+ metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+ Assert.assertEquals(0, metrics.getNumInterruptedBufferWaits());
+
+ // Restart cache and check that right number of entries are read
+ CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher newFlusher =
+ new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, newMetrics);
+ CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(newFlusher)
+ .setCBlockTargetMetrics(newMetrics)
+ .build();
+ newCache.start();
+ Thread fllushListenerThread = new Thread(newFlusher);
+ fllushListenerThread.setDaemon(true);
+ fllushListenerThread.start();
+
+ Thread.sleep(5000);
+ Assert.assertEquals(metrics.getNumBlockBufferUpdates(),
+ newMetrics.getNumDirtyLogBlockRead());
+ Assert.assertEquals(newMetrics.getNumDirtyLogBlockRead()
+ * (Long.SIZE/ Byte.SIZE), newMetrics.getNumBytesDirtyLogReads());
+ // Now shutdown again, nothing should be flushed
+ newFlusher.shutdown();
+ Assert.assertEquals(0, newMetrics.getNumBlockBufferUpdates());
+ Assert.assertEquals(0, newMetrics.getNumBytesDirtyLogWritten());
+ }
+
+ @Test
+ public void testPeriodicFlush() throws IOException,
+ InterruptedException, TimeoutException{
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ flushTestConfig
+ .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 5, SECONDS);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(createContainerAndGetPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ Thread.sleep(8000);
+ // Ticks will be at 5s, 10s and so on, so this count should be 1
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+ // Nothing pushed to cache, so nothing should be written
+ Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+ cache.close();
+ // After close, another trigger should happen but still no data written
+ Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+ Assert.assertEquals(0, metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+ Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+ }
+
+ @Test
+ public void testSingleBufferFlush() throws IOException,
+ InterruptedException, TimeoutException {
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(createContainerAndGetPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+
+ for (int i = 0; i < 511; i++) {
+ cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+ }
+ // After writing 511 block no flush should happen
+ Assert.assertEquals(0, metrics.getNumBlockBufferFlushTriggered());
+ Assert.assertEquals(0, metrics.getNumBlockBufferFlushCompleted());
+
+
+ // After one more block it should
+ cache.put(512, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+ Thread.sleep(1000);
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+ cache.close();
+ Assert.assertEquals(512 * (Long.SIZE / Byte.SIZE),
+ metrics.getNumBytesDirtyLogWritten());
+ }
+
+ @Test
+ public void testMultipleBuffersFlush() throws IOException,
+ InterruptedException, TimeoutException {
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ flushTestConfig
+ .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 120, SECONDS);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(createContainerAndGetPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < 512; j++) {
+ cache.put(i * 512 + j, data.getBytes(StandardCharsets.UTF_8));
+ }
+ // Flush should be triggered after every 512 block write
+ Assert.assertEquals(i + 1, metrics.getNumBlockBufferFlushTriggered());
+ }
+ Assert.assertEquals(0, metrics.getNumIllegalDirtyLogFiles());
+ Assert.assertEquals(0, metrics.getNumFailedDirtyLogFileDeletes());
+ cache.close();
+ Assert.assertEquals(4 * 512 * (Long.SIZE / Byte.SIZE),
+ metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(5, metrics.getNumBlockBufferFlushTriggered());
+ Assert.assertEquals(4, metrics.getNumBlockBufferFlushCompleted());
+ }
+
+ @Test
+ public void testSingleBlockFlush() throws IOException,
+ InterruptedException, TimeoutException{
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ flushTestConfig
+ .setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+ 5, SECONDS);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(createContainerAndGetPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ cache.put(0, data.getBytes(StandardCharsets.UTF_8));
+ Thread.sleep(8000);
+ // Ticks will be at 5s, 10s and so on, so this count should be 1
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+ // 1 block written to cache, which should be flushed
+ Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+ cache.close();
+ // After close, another trigger should happen but no data should be written
+ Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+ Assert.assertEquals(8, metrics.getNumBytesDirtyLogWritten());
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+ Assert.assertEquals(0, metrics.getNumFailedBlockBufferFlushes());
+ }
+
+ @Test
+ public void testRepeatedBlockWrites() throws IOException,
+ InterruptedException, TimeoutException{
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestBufferManager.class.getSimpleName()
+ + RandomStringUtils.randomNumeric(4));
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(createContainerAndGetPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ Thread fllushListenerThread = new Thread(flusher);
+ fllushListenerThread.setDaemon(true);
+ fllushListenerThread.start();
+ cache.start();
+ for (int i = 0; i < 512; i++) {
+ cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+ }
+ Assert.assertEquals(512, metrics.getNumWriteOps());
+ Assert.assertEquals(512, metrics.getNumBlockBufferUpdates());
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushTriggered());
+ Thread.sleep(5000);
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+
+
+ for (int i = 0; i < 512; i++) {
+ cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+ }
+ Assert.assertEquals(1024, metrics.getNumWriteOps());
+ Assert.assertEquals(1024, metrics.getNumBlockBufferUpdates());
+ Assert.assertEquals(2, metrics.getNumBlockBufferFlushTriggered());
+
+ Thread.sleep(5000);
+ Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+ Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
+ Assert.assertEquals(2, metrics.getNumBlockBufferFlushCompleted());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
new file mode 100644
index 0000000..4139ac6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockConfigurationFields.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Tests if configuration constants documented in ozone-defaults.xml.
+ */
+public class TestCBlockConfigurationFields extends TestConfigurationFieldsBase {
+
+ @Override
+ public void initializeMemberVariables() {
+ xmlFilename = new String("cblock-default.xml");
+ configurationClasses =
+ new Class[] {CBlockConfigKeys.class};
+ errorIfMissingConfigProps = true;
+ errorIfMissingXmlProps = true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
new file mode 100644
index 0000000..d995ba6
--- /dev/null
+++ b/hadoop-cblock/server/src/test/java/org/apache/hadoop/cblock/TestCBlockReadWrite.java
@@ -0,0 +1,377 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
+
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.commons.lang.RandomStringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ozone.MiniOzoneClassicCluster;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.LifeCycleState;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationFactor;
+import org.apache.hadoop.hdsl.protocol.proto.HdslProtos.ReplicationType;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.protocolPB
+ .StorageContainerLocationProtocolClientSideTranslatorPB;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+
+/**
+ * Tests for Cblock read write functionality.
+ */
+public class TestCBlockReadWrite {
+ private final static long GB = 1024 * 1024 * 1024;
+ private final static int KB = 1024;
+ private static MiniOzoneCluster cluster;
+ private static OzoneConfiguration config;
+ private static StorageContainerLocationProtocolClientSideTranslatorPB
+ storageContainerLocationClient;
+ private static XceiverClientManager xceiverClientManager;
+
+ @BeforeClass
+ public static void init() throws IOException {
+ config = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+ config.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ config.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ config.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ cluster = new MiniOzoneClassicCluster.Builder(config)
+ .numDataNodes(1)
+ .setHandlerType(OzoneConsts.OZONE_HANDLER_DISTRIBUTED).build();
+ storageContainerLocationClient = cluster
+ .createStorageContainerLocationClient();
+ xceiverClientManager = new XceiverClientManager(config);
+ }
+
+ @AfterClass
+ public static void shutdown() throws InterruptedException {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ IOUtils.cleanupWithLogger(null, storageContainerLocationClient, cluster);
+ }
+
+ /**
+ * getContainerPipelines creates a set of containers and returns the
+ * Pipelines that define those containers.
+ *
+ * @param count - Number of containers to create.
+ * @return - List of Pipelines.
+ * @throws IOException throws Exception
+ */
+ private List<Pipeline> getContainerPipeline(int count) throws IOException {
+ List<Pipeline> containerPipelines = new LinkedList<>();
+ for (int x = 0; x < count; x++) {
+ String traceID = "trace" + RandomStringUtils.randomNumeric(4);
+ String containerName = "container" + RandomStringUtils.randomNumeric(10);
+ Pipeline pipeline =
+ storageContainerLocationClient.allocateContainer(
+ xceiverClientManager.getType(),
+ xceiverClientManager.getFactor(), containerName, "CBLOCK");
+ XceiverClientSpi client = xceiverClientManager.acquireClient(pipeline);
+ ContainerProtocolCalls.createContainer(client, traceID);
+ // This step is needed since we set private data on pipelines, when we
+ // read the list from CBlockServer. So we mimic that action here.
+ pipeline.setData(Longs.toByteArray(x));
+ containerPipelines.add(pipeline);
+ }
+ return containerPipelines;
+ }
+
+ /**
+ * This test creates a cache and performs a simple write / read.
+ * The operations are done by bypassing the cache.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testDirectIO() throws IOException,
+ InterruptedException, TimeoutException {
+ OzoneConfiguration cConfig = new OzoneConfiguration();
+ cConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+ cConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ final long blockID = 0;
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+ String dataHash = DigestUtils.sha256Hex(data);
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(cConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(cConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ Assert.assertFalse(cache.isShortCircuitIOEnabled());
+ cache.put(blockID, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(1, metrics.getNumDirectBlockWrites());
+ Assert.assertEquals(1, metrics.getNumWriteOps());
+ // Please note that this read is directly from remote container
+ LogicalBlock block = cache.get(blockID);
+ Assert.assertEquals(1, metrics.getNumReadOps());
+ Assert.assertEquals(0, metrics.getNumReadCacheHits());
+ Assert.assertEquals(1, metrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+ Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
+
+ cache.put(blockID + 1, data.getBytes(StandardCharsets.UTF_8));
+ Assert.assertEquals(2, metrics.getNumDirectBlockWrites());
+ Assert.assertEquals(2, metrics.getNumWriteOps());
+ Assert.assertEquals(0, metrics.getNumFailedDirectBlockWrites());
+ // Please note that this read is directly from remote container
+ block = cache.get(blockID + 1);
+ Assert.assertEquals(2, metrics.getNumReadOps());
+ Assert.assertEquals(0, metrics.getNumReadCacheHits());
+ Assert.assertEquals(2, metrics.getNumReadCacheMiss());
+ Assert.assertEquals(0, metrics.getNumReadLostBlocks());
+ String readHash = DigestUtils.sha256Hex(block.getData().array());
+ Assert.assertEquals("File content does not match.", dataHash, readHash);
+ GenericTestUtils.waitFor(() -> !cache.isDirtyCache(), 100, 20 * 1000);
+ cache.close();
+ }
+
+ /**
+ * This test writes some block to the cache and then shuts down the cache
+ * The cache is then restarted with "short.circuit.io" disable to check
+ * that the blocks are read correctly from the container.
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testContainerWrites() throws IOException,
+ InterruptedException, TimeoutException {
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL, 3,
+ TimeUnit.SECONDS);
+ XceiverClientManager xcm = new XceiverClientManager(flushTestConfig);
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+
+ int numUniqueBlocks = 4;
+ String[] data = new String[numUniqueBlocks];
+ String[] dataHash = new String[numUniqueBlocks];
+ for (int i = 0; i < numUniqueBlocks; i++) {
+ data[i] = RandomStringUtils.random(4 * KB);
+ dataHash[i] = DigestUtils.sha256Hex(data[i]);
+ }
+
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xcm, metrics);
+ List<Pipeline> pipelines = getContainerPipeline(10);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xcm)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ Thread flushListenerThread = new Thread(flusher);
+ flushListenerThread.setDaemon(true);
+ flushListenerThread.start();
+ Assert.assertTrue(cache.isShortCircuitIOEnabled());
+ // Write data to the cache
+ for (int i = 0; i < 512; i++) {
+ cache.put(i, data[i % numUniqueBlocks].getBytes(StandardCharsets.UTF_8));
+ }
+ // Close the cache and flush the data to the containers
+ cache.close();
+ Assert.assertEquals(0, metrics.getNumDirectBlockWrites());
+ Assert.assertEquals(512, metrics.getNumWriteOps());
+ Thread.sleep(3000);
+ flusher.shutdown();
+ Assert.assertTrue(metrics.getNumBlockBufferFlushTriggered() > 1);
+ Assert.assertEquals(1, metrics.getNumBlockBufferFlushCompleted());
+ Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+ Assert.assertEquals(0, metrics.getNumWriteGenericExceptionRetryBlocks());
+ Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
+ // Now disable DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO and restart cache
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, false);
+ CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher newFlusher =
+ new ContainerCacheFlusher(flushTestConfig, xcm, newMetrics);
+ CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(pipelines)
+ .setClientManager(xcm)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(newFlusher)
+ .setCBlockTargetMetrics(newMetrics)
+ .build();
+ newCache.start();
+ Assert.assertFalse(newCache.isShortCircuitIOEnabled());
+ // this read will be from the container, also match the hash
+ for (int i = 0; i < 512; i++) {
+ LogicalBlock block = newCache.get(i);
+ String readHash = DigestUtils.sha256Hex(block.getData().array());
+ Assert.assertEquals("File content does not match, for index:"
+ + i, dataHash[i % numUniqueBlocks], readHash);
+ }
+ Assert.assertEquals(0, newMetrics.getNumReadLostBlocks());
+ Assert.assertEquals(0, newMetrics.getNumFailedReadBlocks());
+ newCache.close();
+ newFlusher.shutdown();
+ }
+
+ @Test
+ public void testRetryLog() throws IOException,
+ InterruptedException, TimeoutException {
+ // Create a new config so that this tests write metafile to new location
+ OzoneConfiguration flushTestConfig = new OzoneConfiguration();
+ String path = GenericTestUtils
+ .getTempPath(TestCBlockReadWrite.class.getSimpleName());
+ flushTestConfig.set(DFS_CBLOCK_DISK_CACHE_PATH_KEY, path);
+ flushTestConfig.setBoolean(DFS_CBLOCK_TRACE_IO, true);
+ flushTestConfig.setBoolean(DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO, true);
+ flushTestConfig.setTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+ 3,
+ TimeUnit.SECONDS);
+
+ int numblocks = 10;
+ flushTestConfig.setInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE, numblocks);
+
+ String volumeName = "volume" + RandomStringUtils.randomNumeric(4);
+ String userName = "user" + RandomStringUtils.randomNumeric(4);
+ String data = RandomStringUtils.random(4 * KB);
+
+ List<Pipeline> fakeContainerPipelines = new LinkedList<>();
+ PipelineChannel pipelineChannel = new PipelineChannel("fake",
+ LifeCycleState.OPEN, ReplicationType.STAND_ALONE, ReplicationFactor.ONE,
+ "fake");
+ Pipeline fakePipeline = new Pipeline("fake", pipelineChannel);
+ fakePipeline.setData(Longs.toByteArray(1));
+ fakeContainerPipelines.add(fakePipeline);
+
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher flusher = new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, metrics);
+ CBlockLocalCache cache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(fakeContainerPipelines)
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(flusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ cache.start();
+ Thread flushListenerThread = new Thread(flusher);
+ flushListenerThread.setDaemon(true);
+ flushListenerThread.start();
+
+ for (int i = 0; i < numblocks; i++) {
+ cache.put(i, data.getBytes(StandardCharsets.UTF_8));
+ }
+ Assert.assertEquals(numblocks, metrics.getNumWriteOps());
+ Thread.sleep(3000);
+
+ // all the writes to the container will fail because of fake pipelines
+ Assert.assertEquals(numblocks, metrics.getNumDirtyLogBlockRead());
+ Assert.assertTrue(
+ metrics.getNumWriteGenericExceptionRetryBlocks() >= numblocks);
+ Assert.assertEquals(0, metrics.getNumWriteIOExceptionRetryBlocks());
+ Assert.assertEquals(0, metrics.getNumFailedRetryLogFileWrites());
+ Assert.assertEquals(0, metrics.getNumFailedReleaseLevelDB());
+ cache.close();
+ flusher.shutdown();
+
+ // restart cache with correct pipelines, now blocks should be uploaded
+ // correctly
+ CBlockTargetMetrics newMetrics = CBlockTargetMetrics.create();
+ ContainerCacheFlusher newFlusher =
+ new ContainerCacheFlusher(flushTestConfig,
+ xceiverClientManager, newMetrics);
+ CBlockLocalCache newCache = CBlockLocalCache.newBuilder()
+ .setConfiguration(flushTestConfig)
+ .setVolumeName(volumeName)
+ .setUserName(userName)
+ .setPipelines(getContainerPipeline(10))
+ .setClientManager(xceiverClientManager)
+ .setBlockSize(4 * KB)
+ .setVolumeSize(50 * GB)
+ .setFlusher(newFlusher)
+ .setCBlockTargetMetrics(newMetrics)
+ .build();
+ newCache.start();
+ Thread newFlushListenerThread = new Thread(newFlusher);
+ newFlushListenerThread.setDaemon(true);
+ newFlushListenerThread.start();
+ Thread.sleep(3000);
+ Assert.assertTrue(newMetrics.getNumRetryLogBlockRead() >= numblocks);
+ Assert.assertEquals(0, newMetrics.getNumWriteGenericExceptionRetryBlocks());
+ Assert.assertEquals(0, newMetrics.getNumWriteIOExceptionRetryBlocks());
+ Assert.assertEquals(0, newMetrics.getNumFailedReleaseLevelDB());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
deleted file mode 100644
index d0ac974..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/ListContainerHandler.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.cli.container;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.hadoop.ozone.scm.cli.OzoneCommandHandler;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.CMD_WIDTH;
-import static org.apache.hadoop.ozone.scm.cli.SCMCLI.HELP_OP;
-
-/**
- * This is the handler that process container list command.
- */
-public class ListContainerHandler extends OzoneCommandHandler {
-
- public static final String CONTAINER_LIST = "list";
- public static final String OPT_START_CONTAINER = "start";
- public static final String OPT_PREFIX_CONTAINER = "prefix";
- public static final String OPT_COUNT = "count";
-
- /**
- * Constructs a handler object.
- *
- * @param scmClient scm client
- */
- public ListContainerHandler(ScmClient scmClient) {
- super(scmClient);
- }
-
- @Override
- public void execute(CommandLine cmd) throws IOException {
- if (!cmd.hasOption(CONTAINER_LIST)) {
- throw new IOException("Expecting container list");
- }
- if (cmd.hasOption(HELP_OP)) {
- displayHelp();
- return;
- }
-
- if (!cmd.hasOption(OPT_COUNT)) {
- displayHelp();
- if (!cmd.hasOption(HELP_OP)) {
- throw new IOException("Expecting container count");
- } else {
- return;
- }
- }
-
- String startName = cmd.getOptionValue(OPT_START_CONTAINER);
- String prefixName = cmd.getOptionValue(OPT_PREFIX_CONTAINER);
- int count = 0;
-
- if (cmd.hasOption(OPT_COUNT)) {
- count = Integer.parseInt(cmd.getOptionValue(OPT_COUNT));
- if (count < 0) {
- displayHelp();
- throw new IOException("-count should not be negative");
- }
- }
-
- List<ContainerInfo> containerList =
- getScmClient().listContainer(startName, prefixName, count);
-
- // Output data list
- for (ContainerInfo container : containerList) {
- outputContainerPipeline(container.getPipeline());
- }
- }
-
- private void outputContainerPipeline(Pipeline pipeline) throws IOException {
- // Print container report info.
- logOut("%s", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- pipeline.toJsonString()));
- }
-
- @Override
- public void displayHelp() {
- Options options = new Options();
- addOptions(options);
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(CMD_WIDTH, "hdfs scm -container -list <option>",
- "where <option> can be the following", options, "");
- }
-
- public static void addOptions(Options options) {
- Option startContainerOpt = new Option(OPT_START_CONTAINER,
- true, "Specify start container name");
- Option endContainerOpt = new Option(OPT_PREFIX_CONTAINER,
- true, "Specify prefix container name");
- Option countOpt = new Option(OPT_COUNT, true,
- "Specify count number, required");
- options.addOption(countOpt);
- options.addOption(startContainerOpt);
- options.addOption(endContainerOpt);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
deleted file mode 100644
index 1c9f40d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/container/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli.container;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
deleted file mode 100644
index d1f9775..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/cli/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.cli;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
deleted file mode 100644
index 5950d9a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerMapping.java
+++ /dev/null
@@ -1,606 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * <p>http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * <p>Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.lease.Lease;
-import org.apache.hadoop.ozone.lease.LeaseException;
-import org.apache.hadoop.ozone.lease.LeaseManager;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.container.closer.ContainerCloser;
-import org.apache.hadoop.ozone.scm.container.replication.ContainerSupervisor;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-import static org.apache.hadoop.ozone.OzoneConsts.SCM_CONTAINER_DB;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_SIZE_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This
- * is used by SCM when
- * allocating new locations and when looking up a key.
- */
-public class ContainerMapping implements Mapping {
- private static final Logger LOG = LoggerFactory.getLogger(ContainerMapping
- .class);
-
- private final NodeManager nodeManager;
- private final long cacheSize;
- private final Lock lock;
- private final Charset encoding = Charset.forName("UTF-8");
- private final MetadataStore containerStore;
- private final PipelineSelector pipelineSelector;
- private final ContainerStateManager containerStateManager;
- private final LeaseManager<ContainerInfo> containerLeaseManager;
- private final ContainerSupervisor containerSupervisor;
- private final float containerCloseThreshold;
- private final ContainerCloser closer;
- private final long size;
-
- /**
- * Constructs a mapping class that creates mapping between container names
- * and pipelines.
- *
- * @param nodeManager - NodeManager so that we can get the nodes that are
- * healthy to place new
- * containers.
- * @param cacheSizeMB - Amount of memory reserved for the LSM tree to cache
- * its nodes. This is
- * passed to LevelDB and this memory is allocated in Native code space.
- * CacheSize is specified
- * in MB.
- * @throws IOException on Failure.
- */
- @SuppressWarnings("unchecked")
- public ContainerMapping(
- final Configuration conf, final NodeManager nodeManager, final int
- cacheSizeMB) throws IOException {
- this.nodeManager = nodeManager;
- this.cacheSize = cacheSizeMB;
- this.closer = new ContainerCloser(nodeManager, conf);
-
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
-
- // Write the container name to pipeline mapping.
- File containerDBPath = new File(metaDir, SCM_CONTAINER_DB);
- containerStore =
- MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(containerDBPath)
- .setCacheSize(this.cacheSize * OzoneConsts.MB)
- .build();
-
- this.lock = new ReentrantLock();
-
- this.pipelineSelector = new PipelineSelector(nodeManager, conf);
-
- // To be replaced with code getStorageSize once it is committed.
- size = conf.getLong(OZONE_SCM_CONTAINER_SIZE_GB,
- OZONE_SCM_CONTAINER_SIZE_DEFAULT) * 1024 * 1024 * 1024;
- this.containerStateManager =
- new ContainerStateManager(conf, this);
- this.containerSupervisor =
- new ContainerSupervisor(conf, nodeManager,
- nodeManager.getNodePoolManager());
- this.containerCloseThreshold = conf.getFloat(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD,
- ScmConfigKeys.OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT);
- LOG.trace("Container State Manager created.");
-
- long containerCreationLeaseTimeout = conf.getTimeDuration(
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT,
- ScmConfigKeys.OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- LOG.trace("Starting Container Lease Manager.");
- containerLeaseManager = new LeaseManager<>(containerCreationLeaseTimeout);
- containerLeaseManager.start();
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public ContainerInfo getContainer(final String containerName) throws
- IOException {
- ContainerInfo containerInfo;
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(containerName.getBytes(
- encoding));
- if (containerBytes == null) {
- throw new SCMException(
- "Specified key does not exist. key : " + containerName,
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
-
- OzoneProtos.SCMContainerInfo temp = OzoneProtos.SCMContainerInfo.PARSER
- .parseFrom(containerBytes);
- containerInfo = ContainerInfo.fromProtobuf(temp);
- return containerInfo;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName,
- String prefixName, int count) throws IOException {
- List<ContainerInfo> containerList = new ArrayList<>();
- lock.lock();
- try {
- if (containerStore.isEmpty()) {
- throw new IOException("No container exists in current db");
- }
- MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefixName);
- byte[] startKey = startName == null ? null : DFSUtil.string2Bytes(
- startName);
- List<Map.Entry<byte[], byte[]>> range =
- containerStore.getSequentialRangeKVs(startKey, count, prefixFilter);
-
- // Transform the values into the pipelines.
- // TODO: filter by container state
- for (Map.Entry<byte[], byte[]> entry : range) {
- ContainerInfo containerInfo =
- ContainerInfo.fromProtobuf(
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(
- entry.getValue()));
- Preconditions.checkNotNull(containerInfo);
- containerList.add(containerInfo);
- }
- } finally {
- lock.unlock();
- }
- return containerList;
- }
-
- /**
- * Allocates a new container.
- *
- * @param replicationFactor - replication factor of the container.
- * @param containerName - Name of the container.
- * @param owner - The string name of the Service that owns this container.
- * @return - Pipeline that makes up this container.
- * @throws IOException - Exception
- */
- @Override
- public ContainerInfo allocateContainer(
- ReplicationType type,
- ReplicationFactor replicationFactor,
- final String containerName,
- String owner)
- throws IOException {
- Preconditions.checkNotNull(containerName);
- Preconditions.checkState(!containerName.isEmpty());
-
- ContainerInfo containerInfo;
- if (!nodeManager.isOutOfChillMode()) {
- throw new SCMException(
- "Unable to create container while in chill mode",
- SCMException.ResultCodes.CHILL_MODE_EXCEPTION);
- }
-
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(containerName.getBytes(
- encoding));
- if (containerBytes != null) {
- throw new SCMException(
- "Specified container already exists. key : " + containerName,
- SCMException.ResultCodes.CONTAINER_EXISTS);
- }
- containerInfo =
- containerStateManager.allocateContainer(
- pipelineSelector, type, replicationFactor, containerName,
- owner);
- containerStore.put(
- containerName.getBytes(encoding), containerInfo.getProtobuf()
- .toByteArray());
- } finally {
- lock.unlock();
- }
- return containerInfo;
- }
-
- /**
- * Deletes a container from SCM.
- *
- * @param containerName - Container name
- * @throws IOException if container doesn't exist or container store failed
- * to delete the
- * specified key.
- */
- @Override
- public void deleteContainer(String containerName) throws IOException {
- lock.lock();
- try {
- byte[] dbKey = containerName.getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes == null) {
- throw new SCMException(
- "Failed to delete container " + containerName + ", reason : " +
- "container doesn't exist.",
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
- containerStore.delete(dbKey);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc} Used by client to update container state on SCM.
- */
- @Override
- public OzoneProtos.LifeCycleState updateContainerState(
- String containerName, OzoneProtos.LifeCycleEvent event) throws
- IOException {
- ContainerInfo containerInfo;
- lock.lock();
- try {
- byte[] dbKey = containerName.getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes == null) {
- throw new SCMException(
- "Failed to update container state"
- + containerName
- + ", reason : container doesn't exist.",
- SCMException.ResultCodes.FAILED_TO_FIND_CONTAINER);
- }
- containerInfo =
- ContainerInfo.fromProtobuf(OzoneProtos.SCMContainerInfo.PARSER
- .parseFrom(containerBytes));
-
- Preconditions.checkNotNull(containerInfo);
- switch (event) {
- case CREATE:
- // Acquire lease on container
- Lease<ContainerInfo> containerLease =
- containerLeaseManager.acquire(containerInfo);
- // Register callback to be executed in case of timeout
- containerLease.registerCallBack(() -> {
- updateContainerState(containerName,
- OzoneProtos.LifeCycleEvent.TIMEOUT);
- return null;
- });
- break;
- case CREATED:
- // Release the lease on container
- containerLeaseManager.release(containerInfo);
- break;
- case FINALIZE:
- // TODO: we don't need a lease manager here for closing as the
- // container report will include the container state after HDFS-13008
- // If a client failed to update the container close state, DN container
- // report from 3 DNs will be used to close the container eventually.
- break;
- case CLOSE:
- break;
- case UPDATE:
- break;
- case DELETE:
- break;
- case TIMEOUT:
- break;
- case CLEANUP:
- break;
- default:
- throw new SCMException("Unsupported container LifeCycleEvent.",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- // If the below updateContainerState call fails, we should revert the
- // changes made in switch case.
- // Like releasing the lease in case of BEGIN_CREATE.
- ContainerInfo updatedContainer = containerStateManager
- .updateContainerState(containerInfo, event);
- containerStore.put(dbKey, updatedContainer.getProtobuf().toByteArray());
- return updatedContainer.getState();
- } catch (LeaseException e) {
- throw new IOException("Lease Exception.", e);
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Returns the container State Manager.
- *
- * @return ContainerStateManager
- */
- @Override
- public ContainerStateManager getStateManager() {
- return containerStateManager;
- }
-
- /**
- * Process container report from Datanode.
- * <p>
- * Processing follows a very simple logic for time being.
- * <p>
- * 1. Datanodes report the current State -- denoted by the datanodeState
- * <p>
- * 2. We are the older SCM state from the Database -- denoted by
- * the knownState.
- * <p>
- * 3. We copy the usage etc. from currentState to newState and log that
- * newState to the DB. This allows us SCM to bootup again and read the
- * state of the world from the DB, and then reconcile the state from
- * container reports, when they arrive.
- *
- * @param reports Container report
- */
- @Override
- public void processContainerReports(ContainerReportsRequestProto reports)
- throws IOException {
- List<StorageContainerDatanodeProtocolProtos.ContainerInfo>
- containerInfos = reports.getReportsList();
- containerSupervisor.handleContainerReport(reports);
- for (StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState :
- containerInfos) {
- byte[] dbKey = datanodeState.getContainerNameBytes().toByteArray();
- lock.lock();
- try {
- byte[] containerBytes = containerStore.get(dbKey);
- if (containerBytes != null) {
- OzoneProtos.SCMContainerInfo knownState =
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
-
- OzoneProtos.SCMContainerInfo newState =
- reconcileState(datanodeState, knownState);
-
- // FIX ME: This can be optimized, we write twice to memory, where a
- // single write would work well.
- //
- // We need to write this to DB again since the closed only write
- // the updated State.
- containerStore.put(dbKey, newState.toByteArray());
-
- // If the container is closed, then state is already written to SCM
- // DB.TODO: So can we can write only once to DB.
- if (closeContainerIfNeeded(newState)) {
- LOG.info("Closing the Container: {}", newState.getContainerName());
- }
- } else {
- // Container not found in our container db.
- LOG.error("Error while processing container report from datanode :" +
- " {}, for container: {}, reason: container doesn't exist in" +
- "container database.", reports.getDatanodeID(),
- datanodeState.getContainerName());
- }
- } finally {
- lock.unlock();
- }
- }
- }
-
- /**
- * Reconciles the state from Datanode with the state in SCM.
- *
- * @param datanodeState - State from the Datanode.
- * @param knownState - State inside SCM.
- * @return new SCM State for this container.
- */
- private OzoneProtos.SCMContainerInfo reconcileState(
- StorageContainerDatanodeProtocolProtos.ContainerInfo datanodeState,
- OzoneProtos.SCMContainerInfo knownState) {
- OzoneProtos.SCMContainerInfo.Builder builder =
- OzoneProtos.SCMContainerInfo.newBuilder();
- builder.setContainerName(knownState.getContainerName());
- builder.setPipeline(knownState.getPipeline());
- // If used size is greater than allocated size, we will be updating
- // allocated size with used size. This update is done as a fallback
- // mechanism in case SCM crashes without properly updating allocated
- // size. Correct allocated value will be updated by
- // ContainerStateManager during SCM shutdown.
- long usedSize = datanodeState.getUsed();
- long allocated = knownState.getAllocatedBytes() > usedSize ?
- knownState.getAllocatedBytes() : usedSize;
- builder.setAllocatedBytes(allocated);
- builder.setUsedBytes(usedSize);
- builder.setNumberOfKeys(datanodeState.getKeyCount());
- builder.setState(knownState.getState());
- builder.setStateEnterTime(knownState.getStateEnterTime());
- builder.setContainerID(knownState.getContainerID());
- if (knownState.getOwner() != null) {
- builder.setOwner(knownState.getOwner());
- }
- return builder.build();
- }
-
- /**
- * Queues the close container command, to datanode and writes the new state
- * to container DB.
- * <p>
- * TODO : Remove this 2 ContainerInfo definitions. It is brain dead to have
- * one protobuf in one file and another definition in another file.
- *
- * @param newState - This is the state we maintain in SCM.
- * @throws IOException
- */
- private boolean closeContainerIfNeeded(OzoneProtos.SCMContainerInfo newState)
- throws IOException {
- float containerUsedPercentage = 1.0f *
- newState.getUsedBytes() / this.size;
-
- ContainerInfo scmInfo = getContainer(newState.getContainerName());
- if (containerUsedPercentage >= containerCloseThreshold
- && !isClosed(scmInfo)) {
- // We will call closer till get to the closed state.
- // That is SCM will make this call repeatedly until we reach the closed
- // state.
- closer.close(newState);
-
- if (shouldClose(scmInfo)) {
- // This event moves the Container from Open to Closing State, this is
- // a state inside SCM. This is the desired state that SCM wants this
- // container to reach. We will know that a container has reached the
- // closed state from container reports. This state change should be
- // invoked once and only once.
- OzoneProtos.LifeCycleState state = updateContainerState(
- scmInfo.getContainerName(),
- OzoneProtos.LifeCycleEvent.FINALIZE);
- if (state != OzoneProtos.LifeCycleState.CLOSING) {
- LOG.error("Failed to close container {}, reason : Not able " +
- "to " +
- "update container state, current container state: {}.",
- newState.getContainerName(), state);
- return false;
- }
- return true;
- }
- }
- return false;
- }
-
- /**
- * In Container is in closed state, if it is in closed, Deleting or Deleted
- * State.
- *
- * @param info - ContainerInfo.
- * @return true if is in open state, false otherwise
- */
- private boolean shouldClose(ContainerInfo info) {
- return info.getState() == OzoneProtos.LifeCycleState.OPEN;
- }
-
- private boolean isClosed(ContainerInfo info) {
- return info.getState() == OzoneProtos.LifeCycleState.CLOSED;
- }
-
- @VisibleForTesting
- public ContainerCloser getCloser() {
- return closer;
- }
-
- /**
- * Closes this stream and releases any system resources associated with it.
- * If the stream is
- * already closed then invoking this method has no effect.
- * <p>
- * <p>As noted in {@link AutoCloseable#close()}, cases where the close may
- * fail require careful
- * attention. It is strongly advised to relinquish the underlying resources
- * and to internally
- * <em>mark</em> the {@code Closeable} as closed, prior to throwing the
- * {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- if (containerLeaseManager != null) {
- containerLeaseManager.shutdown();
- }
- if (containerStateManager != null) {
- flushContainerInfo();
- containerStateManager.close();
- }
- if (containerStore != null) {
- containerStore.close();
- }
- }
-
- /**
- * Since allocatedBytes of a container is only in memory, stored in
- * containerStateManager, when closing ContainerMapping, we need to update
- * this in the container store.
- *
- * @throws IOException on failure.
- */
- @VisibleForTesting
- public void flushContainerInfo() throws IOException {
- List<ContainerInfo> containers = containerStateManager.getAllContainers();
- List<String> failedContainers = new ArrayList<>();
- for (ContainerInfo info : containers) {
- // even if some container updated failed, others can still proceed
- try {
- byte[] dbKey = info.getContainerName().getBytes(encoding);
- byte[] containerBytes = containerStore.get(dbKey);
- // TODO : looks like when a container is deleted, the container is
- // removed from containerStore but not containerStateManager, so it can
- // return info of a deleted container. may revisit this in the future,
- // for now, just skip a not-found container
- if (containerBytes != null) {
- OzoneProtos.SCMContainerInfo oldInfoProto =
- OzoneProtos.SCMContainerInfo.PARSER.parseFrom(containerBytes);
- ContainerInfo oldInfo = ContainerInfo.fromProtobuf(oldInfoProto);
- ContainerInfo newInfo = new ContainerInfo.Builder()
- .setAllocatedBytes(info.getAllocatedBytes())
- .setContainerName(oldInfo.getContainerName())
- .setNumberOfKeys(oldInfo.getNumberOfKeys())
- .setOwner(oldInfo.getOwner())
- .setPipeline(oldInfo.getPipeline())
- .setState(oldInfo.getState())
- .setUsedBytes(oldInfo.getUsedBytes())
- .build();
- containerStore.put(dbKey, newInfo.getProtobuf().toByteArray());
- } else {
- LOG.debug("Container state manager has container {} but not found " +
- "in container store, a deleted container?",
- info.getContainerName());
- }
- } catch (IOException ioe) {
- failedContainers.add(info.getContainerName());
- }
- }
- if (!failedContainers.isEmpty()) {
- throw new IOException("Error in flushing container info from container " +
- "state manager: " + failedContainers);
- }
- }
-
- @VisibleForTesting
- public MetadataStore getContainerStore() {
- return containerStore;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
deleted file mode 100644
index c586107..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStateManager.java
+++ /dev/null
@@ -1,451 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleEvent;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerState;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerStateMap;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * A container state manager keeps track of container states and returns
- * containers that match various queries.
- * <p>
- * This state machine is driven by a combination of server and client actions.
- * <p>
- * This is how a create container happens: 1. When a container is created, the
- * Server(or SCM) marks that Container as ALLOCATED state. In this state, SCM
- * has chosen a pipeline for container to live on. However, the container is not
- * created yet. This container along with the pipeline is returned to the
- * client.
- * <p>
- * 2. The client when it sees the Container state as ALLOCATED understands that
- * container needs to be created on the specified pipeline. The client lets the
- * SCM know that saw this flag and is initiating the on the data nodes.
- * <p>
- * This is done by calling into notifyObjectCreation(ContainerName,
- * BEGIN_CREATE) flag. When SCM gets this call, SCM puts the container state
- * into CREATING. All this state means is that SCM told Client to create a
- * container and client saw that request.
- * <p>
- * 3. Then client makes calls to datanodes directly, asking the datanodes to
- * create the container. This is done with the help of pipeline that supports
- * this container.
- * <p>
- * 4. Once the creation of the container is complete, the client will make
- * another call to the SCM, this time specifying the containerName and the
- * COMPLETE_CREATE as the Event.
- * <p>
- * 5. With COMPLETE_CREATE event, the container moves to an Open State. This is
- * the state when clients can write to a container.
- * <p>
- * 6. If the client does not respond with the COMPLETE_CREATE event with a
- * certain time, the state machine times out and triggers a delete operation of
- * the container.
- * <p>
- * Please see the function initializeStateMachine below to see how this looks in
- * code.
- * <p>
- * Reusing existing container :
- * <p>
- * The create container call is not made all the time, the system tries to use
- * open containers as much as possible. So in those cases, it looks thru the
- * list of open containers and will return containers that match the specific
- * signature.
- * <p>
- * Please note : Logically there are 3 separate state machines in the case of
- * containers.
- * <p>
- * The Create State Machine -- Commented extensively above.
- * <p>
- * Open/Close State Machine - Once the container is in the Open State,
- * eventually it will be closed, once sufficient data has been written to it.
- * <p>
- * TimeOut Delete Container State Machine - if the container creating times out,
- * then Container State manager decides to delete the container.
- */
-public class ContainerStateManager implements Closeable {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStateManager.class);
-
- private final StateMachine<OzoneProtos.LifeCycleState,
- OzoneProtos.LifeCycleEvent> stateMachine;
-
- private final long containerSize;
- private final ConcurrentHashMap<ContainerState, ContainerID> lastUsedMap;
- private final ContainerStateMap containers;
- private final AtomicLong containerCount;
-
- /**
- * Constructs a Container State Manager that tracks all containers owned by
- * SCM for the purpose of allocation of blocks.
- * <p>
- * TODO : Add Container Tags so we know which containers are owned by SCM.
- */
- @SuppressWarnings("unchecked")
- public ContainerStateManager(Configuration configuration,
- Mapping containerMapping) {
-
- // Initialize the container state machine.
- Set<OzoneProtos.LifeCycleState> finalStates = new HashSet();
-
- // These are the steady states of a container.
- finalStates.add(LifeCycleState.OPEN);
- finalStates.add(LifeCycleState.CLOSED);
- finalStates.add(LifeCycleState.DELETED);
-
- this.stateMachine = new StateMachine<>(LifeCycleState.ALLOCATED,
- finalStates);
- initializeStateMachine();
-
- this.containerSize = OzoneConsts.GB * configuration.getInt(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
-
- lastUsedMap = new ConcurrentHashMap<>();
- containerCount = new AtomicLong(0);
- containers = new ContainerStateMap();
- loadExistingContainers(containerMapping);
- }
-
- private void loadExistingContainers(Mapping containerMapping) {
-
- List<ContainerInfo> containerList;
- try {
- containerList = containerMapping.listContainer(null,
- null, Integer.MAX_VALUE);
-
- // if there are no container to load, let us return.
- if (containerList == null || containerList.size() == 0) {
- LOG.info("No containers to load for this cluster.");
- return;
- }
- } catch (IOException e) {
- if (!e.getMessage().equals("No container exists in current db")) {
- LOG.error("Could not list the containers", e);
- }
- return;
- }
-
- try {
- long maxID = 0;
- for (ContainerInfo container : containerList) {
- containers.addContainer(container);
-
- if (maxID < container.getContainerID()) {
- maxID = container.getContainerID();
- }
-
- containerCount.set(maxID);
- }
- } catch (SCMException ex) {
- LOG.error("Unable to create a container information. ", ex);
- // Fix me, what is the proper shutdown procedure for SCM ??
- // System.exit(1) // Should we exit here?
- }
- }
-
- /**
- * Return the info of all the containers kept by the in-memory mapping.
- *
- * @return the list of all container info.
- */
- public List<ContainerInfo> getAllContainers() {
- List<ContainerInfo> list = new ArrayList<>();
-
- //No Locking needed since the return value is an immutable map.
- containers.getContainerMap().forEach((key, value) -> list.add(value));
- return list;
- }
-
- /*
- *
- * Event and State Transition Mapping:
- *
- * State: ALLOCATED ---------------> CREATING
- * Event: CREATE
- *
- * State: CREATING ---------------> OPEN
- * Event: CREATED
- *
- * State: OPEN ---------------> CLOSING
- * Event: FINALIZE
- *
- * State: CLOSING ---------------> CLOSED
- * Event: CLOSE
- *
- * State: CLOSED ----------------> DELETING
- * Event: DELETE
- *
- * State: DELETING ----------------> DELETED
- * Event: CLEANUP
- *
- * State: CREATING ---------------> DELETING
- * Event: TIMEOUT
- *
- *
- * Container State Flow:
- *
- * [ALLOCATED]------->[CREATING]--------->[OPEN]---------->[CLOSING]------->[CLOSED]
- * (CREATE) | (CREATED) (FINALIZE) (CLOSE) |
- * | |
- * | |
- * |(TIMEOUT) (DELETE)|
- * | |
- * +------------------> [DELETING] <-------------------+
- * |
- * |
- * (CLEANUP)|
- * |
- * [DELETED]
- */
- private void initializeStateMachine() {
- stateMachine.addTransition(LifeCycleState.ALLOCATED,
- LifeCycleState.CREATING,
- LifeCycleEvent.CREATE);
-
- stateMachine.addTransition(LifeCycleState.CREATING,
- LifeCycleState.OPEN,
- LifeCycleEvent.CREATED);
-
- stateMachine.addTransition(LifeCycleState.OPEN,
- LifeCycleState.CLOSING,
- LifeCycleEvent.FINALIZE);
-
- stateMachine.addTransition(LifeCycleState.CLOSING,
- LifeCycleState.CLOSED,
- LifeCycleEvent.CLOSE);
-
- stateMachine.addTransition(LifeCycleState.CLOSED,
- LifeCycleState.DELETING,
- LifeCycleEvent.DELETE);
-
- stateMachine.addTransition(LifeCycleState.CREATING,
- LifeCycleState.DELETING,
- LifeCycleEvent.TIMEOUT);
-
- stateMachine.addTransition(LifeCycleState.DELETING,
- LifeCycleState.DELETED,
- LifeCycleEvent.CLEANUP);
- }
-
- /**
- * allocates a new container based on the type, replication etc.
- *
- * @param selector -- Pipeline selector class.
- * @param type -- Replication type.
- * @param replicationFactor - Replication replicationFactor.
- * @param containerName - Container Name.
- * @return Container Info.
- * @throws IOException on Failure.
- */
- public ContainerInfo allocateContainer(PipelineSelector selector, OzoneProtos
- .ReplicationType type, OzoneProtos.ReplicationFactor replicationFactor,
- final String containerName, String owner) throws
- IOException {
-
- Pipeline pipeline = selector.getReplicationPipeline(type,
- replicationFactor, containerName);
- ContainerInfo containerInfo = new ContainerInfo.Builder()
- .setContainerName(containerName)
- .setState(OzoneProtos.LifeCycleState.ALLOCATED)
- .setPipeline(pipeline)
- // This is bytes allocated for blocks inside container, not the
- // container size
- .setAllocatedBytes(0)
- .setUsedBytes(0)
- .setNumberOfKeys(0)
- .setStateEnterTime(Time.monotonicNow())
- .setOwner(owner)
- .setContainerID(containerCount.incrementAndGet())
- .build();
- Preconditions.checkNotNull(containerInfo);
- containers.addContainer(containerInfo);
- LOG.trace("New container allocated: {}", containerInfo);
- return containerInfo;
- }
-
- /**
- * Update the Container State to the next state.
- *
- * @param info - ContainerInfo
- * @param event - LifeCycle Event
- * @return Updated ContainerInfo.
- * @throws SCMException on Failure.
- */
- public ContainerInfo updateContainerState(ContainerInfo
- info, OzoneProtos.LifeCycleEvent event) throws SCMException {
- LifeCycleState newState;
- try {
- newState = this.stateMachine.getNextState(info.getState(), event);
- } catch (InvalidStateTransitionException ex) {
- String error = String.format("Failed to update container state %s, " +
- "reason: invalid state transition from state: %s upon " +
- "event: %s.",
- info.getPipeline().getContainerName(), info.getState(), event);
- LOG.error(error);
- throw new SCMException(error, FAILED_TO_CHANGE_CONTAINER_STATE);
- }
-
- // This is a post condition after executing getNextState.
- Preconditions.checkNotNull(newState);
- containers.updateState(info, info.getState(), newState);
- return containers.getContainerInfo(info);
- }
-
- /**
- * Update the container State.
- * @param info - Container Info
- * @return ContainerInfo
- * @throws SCMException - on Error.
- */
- public ContainerInfo updateContainerInfo(ContainerInfo info)
- throws SCMException {
- containers.updateContainerInfo(info);
- return containers.getContainerInfo(info);
- }
-
-
- /**
- * Return a container matching the attributes specified.
- *
- * @param size - Space needed in the Container.
- * @param owner - Owner of the container - A specific nameservice.
- * @param type - Replication Type {StandAlone, Ratis}
- * @param factor - Replication Factor {ONE, THREE}
- * @param state - State of the Container-- {Open, Allocated etc.}
- * @return ContainerInfo, null if there is no match found.
- */
- public ContainerInfo getMatchingContainer(final long size,
- String owner, ReplicationType type, ReplicationFactor factor,
- LifeCycleState state) {
-
- // Find containers that match the query spec, if no match return null.
- NavigableSet<ContainerID> matchingSet =
- containers.getMatchingContainerIDs(state, owner, factor, type);
- if (matchingSet == null || matchingSet.size() == 0) {
- return null;
- }
-
- // Get the last used container and find container above the last used
- // container ID.
- ContainerState key = new ContainerState(owner, type, factor);
- ContainerID lastID = lastUsedMap.get(key);
- if(lastID == null) {
- lastID = matchingSet.first();
- }
-
- // There is a small issue here. The first time, we will skip the first
- // container. But in most cases it will not matter.
- NavigableSet<ContainerID> resultSet = matchingSet.tailSet(lastID, false);
- if (resultSet.size() == 0) {
- resultSet = matchingSet;
- }
-
- ContainerInfo selectedContainer =
- findContainerWithSpace(size, resultSet, owner);
- if (selectedContainer == null) {
-
- // If we did not find any space in the tailSet, we need to look for
- // space in the headset, we need to pass true to deal with the
- // situation that we have a lone container that has space. That is we
- // ignored the last used container under the assumption we can find
- // other containers with space, but if have a single container that is
- // not true. Hence we need to include the last used container as the
- // last element in the sorted set.
-
- resultSet = matchingSet.headSet(lastID, true);
- selectedContainer = findContainerWithSpace(size, resultSet, owner);
- }
- // Update the allocated Bytes on this container.
- if(selectedContainer != null) {
- selectedContainer.updateAllocatedBytes(size);
- }
- return selectedContainer;
-
- }
-
- private ContainerInfo findContainerWithSpace(long size,
- NavigableSet<ContainerID> searchSet, String owner) {
- // Get the container with space to meet our request.
- for (ContainerID id : searchSet) {
- ContainerInfo containerInfo = containers.getContainerInfo(id.getId());
- if (containerInfo.getAllocatedBytes() + size <= this.containerSize) {
- containerInfo.updateLastUsedTime();
-
- ContainerState key = new ContainerState(owner,
- containerInfo.getPipeline().getType(),
- containerInfo.getPipeline().getFactor());
- lastUsedMap.put(key, containerInfo.containerID());
- return containerInfo;
- }
- }
- return null;
- }
-
- /**
- * Returns a set of ContainerIDs that match the Container.
- *
- * @param owner Owner of the Containers.
- * @param type - Replication Type of the containers
- * @param factor - Replication factor of the containers.
- * @param state - Current State, like Open, Close etc.
- * @return Set of containers that match the specific query parameters.
- */
- public NavigableSet<ContainerID> getMatchingContainerIDs(
- String owner, ReplicationType type, ReplicationFactor factor,
- LifeCycleState state) {
- return containers.getMatchingContainerIDs(state, owner,
- factor, type);
- }
-
- @Override
- public void close() throws IOException {
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
deleted file mode 100644
index 1372e7f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerAttribute.java
+++ /dev/null
@@ -1,244 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-
-/**
- * Each Attribute that we manage for a container is maintained as a map.
- * <p>
- * Currently we manage the following attributes for a container.
- * <p>
- * 1. StateMap - LifeCycleState -> Set of ContainerIDs
- * 2. TypeMap - ReplicationType -> Set of ContainerIDs
- * 3. OwnerMap - OwnerNames -> Set of ContainerIDs
- * 4. FactorMap - ReplicationFactor -> Set of ContainerIDs
- * <p>
- * This means that for a cluster size of 750 PB -- we will have around 150
- * Million containers, if we assume 5GB average container size.
- * <p>
- * That implies that these maps will take around 2/3 GB of RAM which will be
- * pinned down in the SCM. This is deemed acceptable since we can tune the
- * container size --say we make it 10GB average size, then we can deal with a
- * cluster size of 1.5 exa bytes with the same metadata in SCMs memory.
- * <p>
- * Please note: **This class is not thread safe**. This used to be thread safe,
- * while bench marking we found that ContainerStateMap would be taking 5
- * locks for a single container insert. If we remove locks in this class,
- * then we are able to perform about 540K operations per second, with the
- * locks in this class it goes down to 246K operations per second. Hence we
- * are going to rely on ContainerStateMap locks to maintain consistency of
- * data in these classes too, since ContainerAttribute is only used by
- * ContainerStateMap class.
- */
-public class ContainerAttribute<T> {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerAttribute.class);
-
- private final Map<T, NavigableSet<ContainerID>> attributeMap;
- private static final NavigableSet<ContainerID> EMPTY_SET = Collections
- .unmodifiableNavigableSet(new TreeSet<>());
-
- /**
- * Creates a Container Attribute map from an existing Map.
- *
- * @param attributeMap - AttributeMap
- */
- public ContainerAttribute(Map<T, NavigableSet<ContainerID>> attributeMap) {
- this.attributeMap = attributeMap;
- }
-
- /**
- * Create an empty Container Attribute map.
- */
- public ContainerAttribute() {
- this.attributeMap = new HashMap<>();
- }
-
- /**
- * Insert or update the value in the Attribute map.
- *
- * @param key - The key to the set where the ContainerID should exist.
- * @param value - Actual Container ID.
- * @throws SCMException - on Error
- */
- public boolean insert(T key, ContainerID value) throws SCMException {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(value);
-
- if (attributeMap.containsKey(key)) {
- if (attributeMap.get(key).add(value)) {
- return true; //we inserted the value as it doesn’t exist in the set.
- } else { // Failure indicates that this ContainerID exists in the Set
- if (!attributeMap.get(key).remove(value)) {
- LOG.error("Failure to remove the object from the Map.Key:{}, " +
- "ContainerID: {}", key, value);
- throw new SCMException("Failure to remove the object from the Map",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- attributeMap.get(key).add(value);
- return true;
- }
- } else {
- // This key does not exist, we need to allocate this key in the map.
- // TODO: Replace TreeSet with FoldedTreeSet from HDFS Utils.
- // Skipping for now, since FoldedTreeSet does not have implementations
- // for headSet and TailSet. We need those calls.
- this.attributeMap.put(key, new TreeSet<>());
- // This should not fail, we just allocated this object.
- attributeMap.get(key).add(value);
- return true;
- }
- }
-
- /**
- * Returns true if have this bucket in the attribute map.
- *
- * @param key - Key to lookup
- * @return true if we have the key
- */
- public boolean hasKey(T key) {
- Preconditions.checkNotNull(key);
- return this.attributeMap.containsKey(key);
- }
-
- /**
- * Returns true if we have the key and the containerID in the bucket.
- *
- * @param key - Key to the bucket
- * @param id - container ID that we want to lookup
- * @return true or false
- */
- public boolean hasContainerID(T key, ContainerID id) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(id);
-
- return this.attributeMap.containsKey(key) &&
- this.attributeMap.get(key).contains(id);
- }
-
- /**
- * Returns true if we have the key and the containerID in the bucket.
- *
- * @param key - Key to the bucket
- * @param id - container ID that we want to lookup
- * @return true or false
- */
- public boolean hasContainerID(T key, int id) {
- return hasContainerID(key, ContainerID.valueof(id));
- }
-
- /**
- * Clears all entries for this key type.
- *
- * @param key - Key that identifies the Set.
- */
- public void clearSet(T key) {
- Preconditions.checkNotNull(key);
-
- if (attributeMap.containsKey(key)) {
- attributeMap.get(key).clear();
- } else {
- LOG.debug("key: {} does not exist in the attributeMap", key);
- }
- }
-
- /**
- * Removes a container ID from the set pointed by the key.
- *
- * @param key - key to identify the set.
- * @param value - Container ID
- */
- public boolean remove(T key, ContainerID value) {
- Preconditions.checkNotNull(key);
- Preconditions.checkNotNull(value);
-
- if (attributeMap.containsKey(key)) {
- if (!attributeMap.get(key).remove(value)) {
- LOG.debug("ContainerID: {} does not exist in the set pointed by " +
- "key:{}", value, key);
- return false;
- }
- return true;
- } else {
- LOG.debug("key: {} does not exist in the attributeMap", key);
- return false;
- }
- }
-
- /**
- * Returns the collection that maps to the given key.
- *
- * @param key - Key to the bucket.
- * @return Underlying Set in immutable form.
- */
- public NavigableSet<ContainerID> getCollection(T key) {
- Preconditions.checkNotNull(key);
-
- if (this.attributeMap.containsKey(key)) {
- return Collections.unmodifiableNavigableSet(this.attributeMap.get(key));
- }
- LOG.debug("No such Key. Key {}", key);
- return EMPTY_SET;
- }
-
- /**
- * Moves a ContainerID from one bucket to another.
- *
- * @param currentKey - Current Key
- * @param newKey - newKey
- * @param value - ContainerID
- * @throws SCMException on Error
- */
- public void update(T currentKey, T newKey, ContainerID value)
- throws SCMException {
- Preconditions.checkNotNull(currentKey);
- Preconditions.checkNotNull(newKey);
-
- boolean removed = false;
- try {
- removed = remove(currentKey, value);
- if (!removed) {
- throw new SCMException("Unable to find key in the current key bucket",
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- insert(newKey, value);
- } catch (SCMException ex) {
- // if we removed the key, insert it back to original bucket, since the
- // next insert failed.
- LOG.error("error in update.", ex);
- if (removed) {
- insert(currentKey, value);
- LOG.trace("reinserted the removed key. {}", currentKey);
- }
- throw ex;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
deleted file mode 100644
index 6c492ff..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerState.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * Class that acts as the container state.
- */
-public class ContainerState {
- private final OzoneProtos.ReplicationType type;
- private final String owner;
- private final OzoneProtos.ReplicationFactor replicationFactor;
-
- /**
- * Constructs a Container Key.
- *
- * @param owner - Container Owners
- * @param type - Replication Type.
- * @param factor - Replication Factors
- */
- public ContainerState(String owner, OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor) {
- this.type = type;
- this.owner = owner;
- this.replicationFactor = factor;
- }
-
-
- public OzoneProtos.ReplicationType getType() {
- return type;
- }
-
- public String getOwner() {
- return owner;
- }
-
- public OzoneProtos.ReplicationFactor getFactor() {
- return replicationFactor;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ContainerState that = (ContainerState) o;
-
- return new EqualsBuilder()
- .append(type, that.type)
- .append(owner, that.owner)
- .append(replicationFactor, that.replicationFactor)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(137, 757)
- .append(type)
- .append(owner)
- .append(replicationFactor)
- .toHashCode();
- }
-
- @Override
- public String toString() {
- return "ContainerKey{" +
- ", type=" + type +
- ", owner=" + owner +
- ", replicationFactor=" + replicationFactor +
- '}';
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
deleted file mode 100644
index eebc6be..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerStateMap.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.util.AutoCloseableLock;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.NavigableSet;
-import java.util.TreeSet;
-
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .CONTAINER_EXISTS;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_CHANGE_CONTAINER_STATE;
-import static org.apache.hadoop.ozone.scm.exceptions.SCMException.ResultCodes
- .FAILED_TO_FIND_CONTAINER;
-
-/**
- * Container State Map acts like a unified map for various attributes that are
- * used to select containers when we need allocated blocks.
- * <p>
- * This class provides the ability to query 4 classes of attributes. They are
- * <p>
- * 1. LifeCycleStates - LifeCycle States of container describe in which state
- * a container is. For example, a container needs to be in Open State for a
- * client to able to write to it.
- * <p>
- * 2. Owners - Each instance of Name service, for example, Namenode of HDFS or
- * Key Space Manager (KSM) of Ozone or CBlockServer -- is an owner. It is
- * possible to have many KSMs for a Ozone cluster and only one SCM. But SCM
- * keeps the data from each KSM in separate bucket, never mixing them. To
- * write data, often we have to find all open containers for a specific owner.
- * <p>
- * 3. ReplicationType - The clients are allowed to specify what kind of
- * replication pipeline they want to use. Each Container exists on top of a
- * pipeline, so we need to get ReplicationType that is specified by the user.
- * <p>
- * 4. ReplicationFactor - The replication factor represents how many copies
- * of data should be made, right now we support 2 different types, ONE
- * Replica and THREE Replica. User can specify how many copies should be made
- * for a ozone key.
- * <p>
- * The most common access pattern of this class is to select a container based
- * on all these parameters, for example, when allocating a block we will
- * select a container that belongs to user1, with Ratis replication which can
- * make 3 copies of data. The fact that we will look for open containers by
- * default and if we cannot find them we will add new containers.
- */
-public class ContainerStateMap {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerStateMap.class);
-
- private final ContainerAttribute<LifeCycleState> lifeCycleStateMap;
- private final ContainerAttribute<String> ownerMap;
- private final ContainerAttribute<ReplicationFactor> factorMap;
- private final ContainerAttribute<ReplicationType> typeMap;
-
- private final Map<ContainerID, ContainerInfo> containerMap;
- private final static NavigableSet<ContainerID> EMPTY_SET =
- Collections.unmodifiableNavigableSet(new TreeSet<>());
-
- // Container State Map lock should be held before calling into
- // Update ContainerAttributes. The consistency of ContainerAttributes is
- // protected by this lock.
- private final AutoCloseableLock autoLock;
-
- /**
- * Create a ContainerStateMap.
- */
- public ContainerStateMap() {
- lifeCycleStateMap = new ContainerAttribute<>();
- ownerMap = new ContainerAttribute<>();
- factorMap = new ContainerAttribute<>();
- typeMap = new ContainerAttribute<>();
- containerMap = new HashMap<>();
- autoLock = new AutoCloseableLock();
-// new InstrumentedLock(getClass().getName(), LOG,
-// new ReentrantLock(),
-// 1000,
-// 300));
- }
-
- /**
- * Adds a ContainerInfo Entry in the ContainerStateMap.
- *
- * @param info - container info
- * @throws SCMException - throws if create failed.
- */
- public void addContainer(ContainerInfo info)
- throws SCMException {
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- ContainerID id = ContainerID.valueof(info.getContainerID());
- if (containerMap.putIfAbsent(id, info) != null) {
- LOG.debug("Duplicate container ID detected. {}", id);
- throw new
- SCMException("Duplicate container ID detected.",
- CONTAINER_EXISTS);
- }
-
- lifeCycleStateMap.insert(info.getState(), id);
- ownerMap.insert(info.getOwner(), id);
- factorMap.insert(info.getPipeline().getFactor(), id);
- typeMap.insert(info.getPipeline().getType(), id);
- LOG.trace("Created container with {} successfully.", id);
- }
- }
-
- /**
- * Returns the latest state of Container from SCM's Container State Map.
- *
- * @param info - ContainerInfo
- * @return ContainerInfo
- */
- public ContainerInfo getContainerInfo(ContainerInfo info) {
- return getContainerInfo(info.getContainerID());
- }
-
- /**
- * Returns the latest state of Container from SCM's Container State Map.
- *
- * @param containerID - int
- * @return container info, if found.
- */
- public ContainerInfo getContainerInfo(long containerID) {
- ContainerID id = new ContainerID(containerID);
- return containerMap.get(id);
- }
-
- /**
- * Returns the full container Map.
- *
- * @return - Map
- */
- public Map<ContainerID, ContainerInfo> getContainerMap() {
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return Collections.unmodifiableMap(containerMap);
- }
- }
-
- /**
- * Just update the container State.
- * @param info ContainerInfo.
- */
- public void updateContainerInfo(ContainerInfo info) throws SCMException {
- Preconditions.checkNotNull(info);
- ContainerInfo currentInfo = null;
- try (AutoCloseableLock lock = autoLock.acquire()) {
- currentInfo = containerMap.get(
- ContainerID.valueof(info.getContainerID()));
-
- if (currentInfo == null) {
- throw new SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
- }
- containerMap.put(info.containerID(), info);
- }
- }
-
- /**
- * Update the State of a container.
- *
- * @param info - ContainerInfo
- * @param currentState - CurrentState
- * @param newState - NewState.
- * @throws SCMException - in case of failure.
- */
- public void updateState(ContainerInfo info, LifeCycleState currentState,
- LifeCycleState newState) throws SCMException {
- Preconditions.checkNotNull(currentState);
- Preconditions.checkNotNull(newState);
-
- ContainerID id = new ContainerID(info.getContainerID());
- ContainerInfo currentInfo = null;
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- currentInfo = containerMap.get(id);
-
- if (currentInfo == null) {
- throw new
- SCMException("No such container.", FAILED_TO_FIND_CONTAINER);
- }
- // We are updating two places before this update is done, these can
- // fail independently, since the code needs to handle it.
-
- // We update the attribute map, if that fails it will throw an exception,
- // so no issues, if we are successful, we keep track of the fact that we
- // have updated the lifecycle state in the map, and update the container
- // state. If this second update fails, we will attempt to roll back the
- // earlier change we did. If the rollback fails, we can be in an
- // inconsistent state,
-
- info.setState(newState);
- containerMap.put(id, info);
- lifeCycleStateMap.update(currentState, newState, id);
- LOG.trace("Updated the container {} to new state. Old = {}, new = " +
- "{}", id, currentState, newState);
- } catch (SCMException ex) {
- LOG.error("Unable to update the container state. {}", ex);
- // we need to revert the change in this attribute since we are not
- // able to update the hash table.
- LOG.info("Reverting the update to lifecycle state. Moving back to " +
- "old state. Old = {}, Attempted state = {}", currentState,
- newState);
-
- containerMap.put(id, currentInfo);
-
- // if this line throws, the state map can be in an inconsistent
- // state, since we will have modified the attribute by the
- // container state will not in sync since we were not able to put
- // that into the hash table.
- lifeCycleStateMap.update(newState, currentState, id);
-
- throw new SCMException("Updating the container map failed.", ex,
- FAILED_TO_CHANGE_CONTAINER_STATE);
- }
- }
-
- /**
- * Returns A list of containers owned by a name service.
- *
- * @param ownerName - Name of the NameService.
- * @return - NavigableSet of ContainerIDs.
- */
- NavigableSet<ContainerID> getContainerIDsByOwner(String ownerName) {
- Preconditions.checkNotNull(ownerName);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return ownerMap.getCollection(ownerName);
- }
- }
-
- /**
- * Returns Containers in the System by the Type.
- *
- * @param type - Replication type -- StandAlone, Ratis etc.
- * @return NavigableSet
- */
- NavigableSet<ContainerID> getContainerIDsByType(ReplicationType type) {
- Preconditions.checkNotNull(type);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return typeMap.getCollection(type);
- }
- }
-
- /**
- * Returns Containers by replication factor.
- *
- * @param factor - Replication Factor.
- * @return NavigableSet.
- */
- NavigableSet<ContainerID> getContainerIDsByFactor(ReplicationFactor factor) {
- Preconditions.checkNotNull(factor);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return factorMap.getCollection(factor);
- }
- }
-
- /**
- * Returns Containers by State.
- *
- * @param state - State - Open, Closed etc.
- * @return List of containers by state.
- */
- NavigableSet<ContainerID> getContainerIDsByState(LifeCycleState state) {
- Preconditions.checkNotNull(state);
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
- return lifeCycleStateMap.getCollection(state);
- }
- }
-
- /**
- * Gets the containers that matches the following filters.
- *
- * @param state - LifeCycleState
- * @param owner - Owner
- * @param factor - Replication Factor
- * @param type - Replication Type
- * @return ContainerInfo or Null if not container satisfies the criteria.
- */
- public NavigableSet<ContainerID> getMatchingContainerIDs(
- LifeCycleState state, String owner,
- ReplicationFactor factor, ReplicationType type) {
-
- Preconditions.checkNotNull(state, "State cannot be null");
- Preconditions.checkNotNull(owner, "Owner cannot be null");
- Preconditions.checkNotNull(factor, "Factor cannot be null");
- Preconditions.checkNotNull(type, "Type cannot be null");
-
- try (AutoCloseableLock lock = autoLock.acquire()) {
-
- // If we cannot meet any one condition we return EMPTY_SET immediately.
- // Since when we intersect these sets, the result will be empty if any
- // one is empty.
- NavigableSet<ContainerID> stateSet =
- lifeCycleStateMap.getCollection(state);
- if (stateSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> ownerSet = ownerMap.getCollection(owner);
- if (ownerSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> factorSet = factorMap.getCollection(factor);
- if (factorSet.size() == 0) {
- return EMPTY_SET;
- }
-
- NavigableSet<ContainerID> typeSet = typeMap.getCollection(type);
- if (typeSet.size() == 0) {
- return EMPTY_SET;
- }
-
-
- // if we add more constraints we will just add those sets here..
- NavigableSet<ContainerID>[] sets = sortBySize(stateSet,
- ownerSet, factorSet, typeSet);
-
- NavigableSet<ContainerID> currentSet = sets[0];
- // We take the smallest set and intersect against the larger sets. This
- // allows us to reduce the lookups to the least possible number.
- for (int x = 1; x < sets.length; x++) {
- currentSet = intersectSets(currentSet, sets[x]);
- }
- return currentSet;
- }
- }
-
- /**
- * Calculates the intersection between sets and returns a new set.
- *
- * @param smaller - First Set
- * @param bigger - Second Set
- * @return resultSet which is the intersection of these two sets.
- */
- private NavigableSet<ContainerID> intersectSets(
- NavigableSet<ContainerID> smaller,
- NavigableSet<ContainerID> bigger) {
- Preconditions.checkState(smaller.size() <= bigger.size(),
- "This function assumes the first set is lesser or equal to second " +
- "set");
- NavigableSet<ContainerID> resultSet = new TreeSet<>();
- for (ContainerID id : smaller) {
- if (bigger.contains(id)) {
- resultSet.add(id);
- }
- }
- return resultSet;
- }
-
- /**
- * Sorts a list of Sets based on Size. This is useful when we are
- * intersecting the sets.
- *
- * @param sets - varagrs of sets
- * @return Returns a sorted array of sets based on the size of the set.
- */
- @SuppressWarnings("unchecked")
- private NavigableSet<ContainerID>[] sortBySize(
- NavigableSet<ContainerID>... sets) {
- for (int x = 0; x < sets.length - 1; x++) {
- for (int y = 0; y < sets.length - x - 1; y++) {
- if (sets[y].size() > sets[y + 1].size()) {
- NavigableSet temp = sets[y];
- sets[y] = sets[y + 1];
- sets[y + 1] = temp;
- }
- }
- }
- return sets;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
deleted file mode 100644
index 6a7e663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * Container States management package.
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
deleted file mode 100644
index 0d442d1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/Mapping.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container;
-
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Mapping class contains the mapping from a name to a pipeline mapping. This is
- * used by SCM when allocating new locations and when looking up a key.
- */
-public interface Mapping extends Closeable {
- /**
- * Returns the ContainerInfo from the container name.
- *
- * @param containerName - Name
- * @return - ContainerInfo such as creation state and the pipeline.
- * @throws IOException
- */
- ContainerInfo getContainer(String containerName) throws IOException;
-
- /**
- * Returns containers under certain conditions.
- * Search container names from start name(exclusive),
- * and use prefix name to filter the result. The max
- * size of the searching range cannot exceed the
- * value of count.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of container.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Allocates a new container for a given keyName and replication factor.
- *
- * @param replicationFactor - replication factor of the container.
- * @param containerName - Name.
- * @param owner
- * @return - Container Info.
- * @throws IOException
- */
- ContainerInfo allocateContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor replicationFactor,
- String containerName, String owner) throws IOException;
-
- /**
- * Deletes a container from SCM.
- *
- * @param containerName - Container Name
- * @throws IOException
- */
- void deleteContainer(String containerName) throws IOException;
-
- /**
- * Update container state.
- * @param containerName - Container Name
- * @param event - container life cycle event
- * @return - new container state
- * @throws IOException
- */
- OzoneProtos.LifeCycleState updateContainerState(String containerName,
- OzoneProtos.LifeCycleEvent event) throws IOException;
-
- /**
- * Returns the container State Manager.
- * @return ContainerStateManager
- */
- ContainerStateManager getStateManager();
-
- /**
- * Process container report from Datanode.
- *
- * @param reports Container report
- */
- void processContainerReports(ContainerReportsRequestProto reports)
- throws IOException;
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
deleted file mode 100644
index d411a6b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ContainerOperationClient.java
+++ /dev/null
@@ -1,403 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ReadContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.UUID;
-
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.ALLOCATED;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState.OPEN;
-
-/**
- * This class provides the client-facing APIs of container operations.
- */
-public class ContainerOperationClient implements ScmClient {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerOperationClient.class);
- private static long containerSizeB = -1;
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final XceiverClientManager xceiverClientManager;
-
- public ContainerOperationClient(
- StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient,
- XceiverClientManager xceiverClientManager) {
- this.storageContainerLocationClient = storageContainerLocationClient;
- this.xceiverClientManager = xceiverClientManager;
- }
-
- /**
- * Return the capacity of containers. The current assumption is that all
- * containers have the same capacity. Therefore one static is sufficient for
- * any container.
- * @return The capacity of one container in number of bytes.
- */
- public static long getContainerSizeB() {
- return containerSizeB;
- }
-
- /**
- * Set the capacity of container. Should be exactly once on system start.
- * @param size Capacity of one container in number of bytes.
- */
- public static void setContainerSizeB(long size) {
- containerSizeB = size;
- }
-
- /**
- * @inheritDoc
- */
- @Override
- public Pipeline createContainer(String containerId, String owner)
- throws IOException {
- XceiverClientSpi client = null;
- try {
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(
- xceiverClientManager.getType(),
- xceiverClientManager.getFactor(), containerId, owner);
- client = xceiverClientManager.acquireClient(pipeline);
-
- // Allocated State means that SCM has allocated this pipeline in its
- // namespace. The client needs to create the pipeline on the machines
- // which was choosen by the SCM.
- Preconditions.checkState(pipeline.getLifeCycleState() == ALLOCATED ||
- pipeline.getLifeCycleState() == OPEN, "Unexpected pipeline state");
- if (pipeline.getLifeCycleState() == ALLOCATED) {
- createPipeline(client, pipeline);
- }
- // TODO : Container Client State needs to be updated.
- // TODO : Return ContainerInfo instead of Pipeline
- createContainer(containerId, client, pipeline);
- return pipeline;
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Create a container over pipeline specified by the SCM.
- *
- * @param containerId - Container ID
- * @param client - Client to communicate with Datanodes
- * @param pipeline - A pipeline that is already created.
- * @throws IOException
- */
- public void createContainer(String containerId, XceiverClientSpi client,
- Pipeline pipeline) throws IOException {
- String traceID = UUID.randomUUID().toString();
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.begin);
- ContainerProtocolCalls.createContainer(client, traceID);
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.complete);
-
- // Let us log this info after we let SCM know that we have completed the
- // creation state.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created container " + containerId
- + " leader:" + pipeline.getLeader()
- + " machines:" + pipeline.getMachines());
- }
- }
-
- /**
- * Creates a pipeline over the machines choosen by the SCM.
- *
- * @param client - Client
- * @param pipeline - pipeline to be createdon Datanodes.
- * @throws IOException
- */
- private void createPipeline(XceiverClientSpi client, Pipeline pipeline)
- throws IOException {
-
- Preconditions.checkNotNull(pipeline.getPipelineName(), "Pipeline " +
- "name cannot be null when client create flag is set.");
-
- // Pipeline creation is a three step process.
- //
- // 1. Notify SCM that this client is doing a create pipeline on
- // datanodes.
- //
- // 2. Talk to Datanodes to create the pipeline.
- //
- // 3. update SCM that pipeline creation was successful.
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.pipeline,
- pipeline.getPipelineName(),
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.begin);
-
- client.createPipeline(pipeline.getPipelineName(),
- pipeline.getMachines());
-
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.pipeline,
- pipeline.getPipelineName(),
- ObjectStageChangeRequestProto.Op.create,
- ObjectStageChangeRequestProto.Stage.complete);
-
- // TODO : Should we change the state on the client side ??
- // That makes sense, but it is not needed for the client to work.
- LOG.debug("Pipeline creation successful. Pipeline: {}",
- pipeline.toString());
- }
-
- /**
- * @inheritDoc
- */
- @Override
- public Pipeline createContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor,
- String containerId, String owner) throws IOException {
- XceiverClientSpi client = null;
- try {
- // allocate container on SCM.
- Pipeline pipeline =
- storageContainerLocationClient.allocateContainer(type, factor,
- containerId, owner);
- client = xceiverClientManager.acquireClient(pipeline);
-
- // Allocated State means that SCM has allocated this pipeline in its
- // namespace. The client needs to create the pipeline on the machines
- // which was choosen by the SCM.
- if (pipeline.getLifeCycleState() == ALLOCATED) {
- createPipeline(client, pipeline);
- }
-
- // TODO : Return ContainerInfo instead of Pipeline
- // connect to pipeline leader and allocate container on leader datanode.
- client = xceiverClientManager.acquireClient(pipeline);
- createContainer(containerId, client, pipeline);
- return pipeline;
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Returns a set of Nodes that meet a query criteria.
- *
- * @param nodeStatuses - A set of criteria that we want the node to have.
- * @param queryScope - Query scope - Cluster or pool.
- * @param poolName - if it is pool, a pool name is required.
- * @return A set of nodes that meet the requested criteria.
- * @throws IOException
- */
- @Override
- public OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState>
- nodeStatuses, OzoneProtos.QueryScope queryScope, String poolName)
- throws IOException {
- return storageContainerLocationClient.queryNode(nodeStatuses, queryScope,
- poolName);
- }
-
- /**
- * Creates a specified replication pipeline.
- */
- @Override
- public Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException {
- return storageContainerLocationClient.createReplicationPipeline(type,
- factor, nodePool);
- }
-
- /**
- * Delete the container, this will release any resource it uses.
- * @param pipeline - Pipeline that represents the container.
- * @param force - True to forcibly delete the container.
- * @throws IOException
- */
- @Override
- public void deleteContainer(Pipeline pipeline, boolean force)
- throws IOException {
- XceiverClientSpi client = null;
- try {
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
- ContainerProtocolCalls.deleteContainer(client, force, traceID);
- storageContainerLocationClient
- .deleteContainer(pipeline.getContainerName());
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleted container {}, leader: {}, machines: {} ",
- pipeline.getContainerName(),
- pipeline.getLeader(),
- pipeline.getMachines());
- }
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<ContainerInfo> listContainer(String startName,
- String prefixName, int count)
- throws IOException {
- return storageContainerLocationClient.listContainer(
- startName, prefixName, count);
- }
-
- /**
- * Get meta data from an existing container.
- *
- * @param pipeline - pipeline that represents the container.
- * @return ContainerInfo - a message of protobuf which has basic info
- * of a container.
- * @throws IOException
- */
- @Override
- public ContainerData readContainer(Pipeline pipeline) throws IOException {
- XceiverClientSpi client = null;
- try {
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
- ReadContainerResponseProto response =
- ContainerProtocolCalls.readContainer(client,
- pipeline.getContainerName(), traceID);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Read container {}, leader: {}, machines: {} ",
- pipeline.getContainerName(),
- pipeline.getLeader(),
- pipeline.getMachines());
- }
- return response.getContainerData();
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Given an id, return the pipeline associated with the container.
- * @param containerId - String Container ID
- * @return Pipeline of the existing container, corresponding to the given id.
- * @throws IOException
- */
- @Override
- public Pipeline getContainer(String containerId) throws
- IOException {
- return storageContainerLocationClient.getContainer(containerId);
- }
-
- /**
- * Close a container.
- *
- * @param pipeline the container to be closed.
- * @throws IOException
- */
- @Override
- public void closeContainer(Pipeline pipeline) throws IOException {
- XceiverClientSpi client = null;
- try {
- LOG.debug("Close container {}", pipeline);
- /*
- TODO: two orders here, revisit this later:
- 1. close on SCM first, then on data node
- 2. close on data node first, then on SCM
-
- with 1: if client failed after closing on SCM, then there is a
- container SCM thinks as closed, but is actually open. Then SCM will no
- longer allocate block to it, which is fine. But SCM may later try to
- replicate this "closed" container, which I'm not sure is safe.
-
- with 2: if client failed after close on datanode, then there is a
- container SCM thinks as open, but is actually closed. Then SCM will still
- try to allocate block to it. Which will fail when actually doing the
- write. No more data can be written, but at least the correctness and
- consistency of existing data will maintain.
-
- For now, take the #2 way.
- */
- // Actually close the container on Datanode
- client = xceiverClientManager.acquireClient(pipeline);
- String traceID = UUID.randomUUID().toString();
-
- String containerId = pipeline.getContainerName();
-
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.close,
- ObjectStageChangeRequestProto.Stage.begin);
-
- ContainerProtocolCalls.closeContainer(client, traceID);
- // Notify SCM to close the container
- storageContainerLocationClient.notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type.container,
- containerId,
- ObjectStageChangeRequestProto.Op.close,
- ObjectStageChangeRequestProto.Stage.complete);
- } finally {
- if (client != null) {
- xceiverClientManager.releaseClient(client);
- }
- }
- }
-
- /**
- * Get the the current usage information.
- * @param pipeline - Pipeline
- * @return the size of the given container.
- * @throws IOException
- */
- @Override
- public long getContainerSize(Pipeline pipeline) throws IOException {
- // TODO : Pipeline can be null, handle it correctly.
- long size = getContainerSizeB();
- if (size == -1) {
- throw new IOException("Container size unknown!");
- }
- return size;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
deleted file mode 100644
index ef2f1e0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/ScmClient.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.client;
-
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-/**
- * The interface to call into underlying container layer.
- *
- * Written as interface to allow easy testing: implement a mock container layer
- * for standalone testing of CBlock API without actually calling into remote
- * containers. Actual container layer can simply re-implement this.
- *
- * NOTE this is temporarily needed class. When SCM containers are full-fledged,
- * this interface will likely be removed.
- */
-@InterfaceStability.Unstable
-public interface ScmClient {
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @param containerId - String container ID
- * @return Pipeline
- * @throws IOException
- */
- Pipeline createContainer(String containerId, String owner) throws IOException;
-
- /**
- * Gets a container by Name -- Throws if the container does not exist.
- * @param containerId - String Container ID
- * @return Pipeline
- * @throws IOException
- */
- Pipeline getContainer(String containerId) throws IOException;
-
- /**
- * Close a container by name.
- *
- * @param pipeline the container to be closed.
- * @throws IOException
- */
- void closeContainer(Pipeline pipeline) throws IOException;
-
- /**
- * Deletes an existing container.
- * @param pipeline - Pipeline that represents the container.
- * @param force - true to forcibly delete the container.
- * @throws IOException
- */
- void deleteContainer(Pipeline pipeline, boolean force) throws IOException;
-
- /**
- * Lists a range of containers and get their info.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of pipeline.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Read meta data from an existing container.
- * @param pipeline - Pipeline that represents the container.
- * @return ContainerInfo
- * @throws IOException
- */
- ContainerData readContainer(Pipeline pipeline) throws IOException;
-
-
- /**
- * Gets the container size -- Computed by SCM from Container Reports.
- * @param pipeline - Pipeline
- * @return number of bytes used by this container.
- * @throws IOException
- */
- long getContainerSize(Pipeline pipeline) throws IOException;
-
- /**
- * Creates a Container on SCM and returns the pipeline.
- * @param type - Replication Type.
- * @param replicationFactor - Replication Factor
- * @param containerId - Container ID
- * @return Pipeline
- * @throws IOException - in case of error.
- */
- Pipeline createContainer(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor replicationFactor, String containerId,
- String owner) throws IOException;
-
- /**
- * Returns a set of Nodes that meet a query criteria.
- * @param nodeStatuses - A set of criteria that we want the node to have.
- * @param queryScope - Query scope - Cluster or pool.
- * @param poolName - if it is pool, a pool name is required.
- * @return A set of nodes that meet the requested criteria.
- * @throws IOException
- */
- OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
- OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
- /**
- * Creates a specified replication pipeline.
- * @param type - Type
- * @param factor - Replication factor
- * @param nodePool - Set of machines.
- * @throws IOException
- */
- Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
deleted file mode 100644
index e85e542..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/client/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.client;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
deleted file mode 100644
index f51336f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/AllocatedBlock.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-/**
- * Allocated block wraps the result returned from SCM#allocateBlock which
- * contains a Pipeline and the key.
- */
-public final class AllocatedBlock {
- private Pipeline pipeline;
- private String key;
- // Indicates whether the client should create container before writing block.
- private boolean shouldCreateContainer;
-
- /**
- * Builder for AllocatedBlock.
- */
- public static class Builder {
- private Pipeline pipeline;
- private String key;
- private boolean shouldCreateContainer;
-
- public Builder setPipeline(Pipeline p) {
- this.pipeline = p;
- return this;
- }
-
- public Builder setKey(String k) {
- this.key = k;
- return this;
- }
-
- public Builder setShouldCreateContainer(boolean shouldCreate) {
- this.shouldCreateContainer = shouldCreate;
- return this;
- }
-
- public AllocatedBlock build() {
- return new AllocatedBlock(pipeline, key, shouldCreateContainer);
- }
- }
-
- private AllocatedBlock(Pipeline pipeline, String key,
- boolean shouldCreateContainer) {
- this.pipeline = pipeline;
- this.key = key;
- this.shouldCreateContainer = shouldCreateContainer;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public String getKey() {
- return key;
- }
-
- public boolean getCreateContainer() {
- return shouldCreateContainer;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
deleted file mode 100644
index 11d4438..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/ContainerInfo.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.lang3.builder.EqualsBuilder;
-import org.apache.commons.lang3.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.container.ContainerStates.ContainerID;
-import org.apache.hadoop.util.Time;
-
-import java.util.Comparator;
-
-/**
- * Class wraps ozone container info.
- */
-public class ContainerInfo
- implements Comparator<ContainerInfo>, Comparable<ContainerInfo> {
- private OzoneProtos.LifeCycleState state;
- private Pipeline pipeline;
- // Bytes allocated by SCM for clients.
- private long allocatedBytes;
- // Actual container usage, updated through heartbeat.
- private long usedBytes;
- private long numberOfKeys;
- private long lastUsed;
- // The wall-clock ms since the epoch at which the current state enters.
- private long stateEnterTime;
- private String owner;
- private String containerName;
- private long containerID;
- ContainerInfo(
- long containerID,
- final String containerName,
- OzoneProtos.LifeCycleState state,
- Pipeline pipeline,
- long allocatedBytes,
- long usedBytes,
- long numberOfKeys,
- long stateEnterTime,
- String owner) {
- this.containerID = containerID;
- this.containerName = containerName;
- this.pipeline = pipeline;
- this.allocatedBytes = allocatedBytes;
- this.usedBytes = usedBytes;
- this.numberOfKeys = numberOfKeys;
- this.lastUsed = Time.monotonicNow();
- this.state = state;
- this.stateEnterTime = stateEnterTime;
- this.owner = owner;
- }
-
- /**
- * Needed for serialization findbugs.
- */
- public ContainerInfo() {
- }
-
- public static ContainerInfo fromProtobuf(OzoneProtos.SCMContainerInfo info) {
- ContainerInfo.Builder builder = new ContainerInfo.Builder();
- builder.setPipeline(Pipeline.getFromProtoBuf(info.getPipeline()));
- builder.setAllocatedBytes(info.getAllocatedBytes());
- builder.setUsedBytes(info.getUsedBytes());
- builder.setNumberOfKeys(info.getNumberOfKeys());
- builder.setState(info.getState());
- builder.setStateEnterTime(info.getStateEnterTime());
- builder.setOwner(info.getOwner());
- builder.setContainerName(info.getContainerName());
- builder.setContainerID(info.getContainerID());
- return builder.build();
- }
-
- public long getContainerID() {
- return containerID;
- }
-
- public String getContainerName() {
- return containerName;
- }
-
- public OzoneProtos.LifeCycleState getState() {
- return state;
- }
-
- public void setState(OzoneProtos.LifeCycleState state) {
- this.state = state;
- }
-
- public long getStateEnterTime() {
- return stateEnterTime;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public long getAllocatedBytes() {
- return allocatedBytes;
- }
-
- /**
- * Set Allocated bytes.
- *
- * @param size - newly allocated bytes -- negative size is case of deletes
- * can be used.
- */
- public void updateAllocatedBytes(long size) {
- this.allocatedBytes += size;
- }
-
- public long getUsedBytes() {
- return usedBytes;
- }
-
- public long getNumberOfKeys() {
- return numberOfKeys;
- }
-
- public ContainerID containerID() {
- return new ContainerID(getContainerID());
- }
-
- /**
- * Gets the last used time from SCM's perspective.
- *
- * @return time in milliseconds.
- */
- public long getLastUsed() {
- return lastUsed;
- }
-
- public void updateLastUsedTime() {
- lastUsed = Time.monotonicNow();
- }
-
- public void allocate(long size) {
- // should we also have total container size in ContainerInfo
- // and check before allocating?
- allocatedBytes += size;
- }
-
- public OzoneProtos.SCMContainerInfo getProtobuf() {
- OzoneProtos.SCMContainerInfo.Builder builder =
- OzoneProtos.SCMContainerInfo.newBuilder();
- builder.setPipeline(getPipeline().getProtobufMessage());
- builder.setAllocatedBytes(getAllocatedBytes());
- builder.setUsedBytes(getUsedBytes());
- builder.setNumberOfKeys(getNumberOfKeys());
- builder.setState(state);
- builder.setStateEnterTime(stateEnterTime);
- builder.setContainerID(getContainerID());
-
- if (getOwner() != null) {
- builder.setOwner(getOwner());
- }
- builder.setContainerName(getContainerName());
- return builder.build();
- }
-
- public String getOwner() {
- return owner;
- }
-
- public void setOwner(String owner) {
- this.owner = owner;
- }
-
- @Override
- public String toString() {
- return "ContainerInfo{"
- + "state=" + state
- + ", pipeline=" + pipeline
- + ", stateEnterTime=" + stateEnterTime
- + ", owner=" + owner
- + ", containerName='" + containerName
- + '}';
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ContainerInfo that = (ContainerInfo) o;
-
- return new EqualsBuilder()
- .append(pipeline.getContainerName(), that.pipeline.getContainerName())
-
- // TODO : Fix this later. If we add these factors some tests fail.
- // So Commenting this to continue and will enforce this with
- // Changes in pipeline where we remove Container Name to
- // SCMContainerinfo from Pipeline.
- // .append(pipeline.getFactor(), that.pipeline.getFactor())
- // .append(pipeline.getType(), that.pipeline.getType())
- .append(owner, that.owner)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(11, 811)
- .append(pipeline.getContainerName())
- .append(pipeline.getFactor())
- .append(pipeline.getType())
- .append(owner)
- .toHashCode();
- }
-
- /**
- * Compares its two arguments for order. Returns a negative integer, zero, or
- * a positive integer as the first argument is less than, equal to, or greater
- * than the second.<p>
- *
- * @param o1 the first object to be compared.
- * @param o2 the second object to be compared.
- * @return a negative integer, zero, or a positive integer as the first
- * argument is less than, equal to, or greater than the second.
- * @throws NullPointerException if an argument is null and this comparator
- * does not permit null arguments
- * @throws ClassCastException if the arguments' types prevent them from
- * being compared by this comparator.
- */
- @Override
- public int compare(ContainerInfo o1, ContainerInfo o2) {
- return Long.compare(o1.getLastUsed(), o2.getLastUsed());
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less than,
- * equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(ContainerInfo o) {
- return this.compare(this, o);
- }
-
- /**
- * Builder class for ContainerInfo.
- */
- public static class Builder {
- private OzoneProtos.LifeCycleState state;
- private Pipeline pipeline;
- private long allocated;
- private long used;
- private long keys;
- private long stateEnterTime;
- private String owner;
- private String containerName;
- private long containerID;
-
- public Builder setContainerID(long id) {
- Preconditions.checkState(id >= 0);
- this.containerID = id;
- return this;
- }
-
- public Builder setState(OzoneProtos.LifeCycleState lifeCycleState) {
- this.state = lifeCycleState;
- return this;
- }
-
- public Builder setPipeline(Pipeline containerPipeline) {
- this.pipeline = containerPipeline;
- return this;
- }
-
- public Builder setAllocatedBytes(long bytesAllocated) {
- this.allocated = bytesAllocated;
- return this;
- }
-
- public Builder setUsedBytes(long bytesUsed) {
- this.used = bytesUsed;
- return this;
- }
-
- public Builder setNumberOfKeys(long keyCount) {
- this.keys = keyCount;
- return this;
- }
-
- public Builder setStateEnterTime(long time) {
- this.stateEnterTime = time;
- return this;
- }
-
- public Builder setOwner(String containerOwner) {
- this.owner = containerOwner;
- return this;
- }
-
- public Builder setContainerName(String container) {
- this.containerName = container;
- return this;
- }
-
- public ContainerInfo build() {
- return new
- ContainerInfo(containerID, containerName, state, pipeline,
- allocated, used, keys, stateEnterTime, owner);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
deleted file mode 100644
index 5dbb853..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/DeleteBlockResult.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-
-import static org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.DeleteScmBlockResult;
-
-/**
- * Class wraps storage container manager block deletion results.
- */
-public class DeleteBlockResult {
- private String key;
- private DeleteScmBlockResult.Result result;
-
- public DeleteBlockResult(final String key,
- final DeleteScmBlockResult.Result result) {
- this.key = key;
- this.result = result;
- }
-
- /**
- * Get key deleted.
- * @return key name.
- */
- public String getKey() {
- return key;
- }
-
- /**
- * Get key deletion result.
- * @return key deletion result.
- */
- public DeleteScmBlockResult.Result getResult() {
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
deleted file mode 100644
index 9d47646..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/Pipeline.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * A pipeline represents the group of machines over which a container lives.
- */
-public class Pipeline {
- static final String PIPELINE_INFO = "PIPELINE_INFO_FILTER";
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"data"};
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(PIPELINE_INFO, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- WRITER = mapper.writer(filters);
- }
-
- private String containerName;
- private PipelineChannel pipelineChannel;
- /**
- * Allows you to maintain private data on pipelines. This is not serialized
- * via protobuf, just allows us to maintain some private data.
- */
- @JsonIgnore
- private byte[] data;
- /**
- * Constructs a new pipeline data structure.
- *
- * @param containerName - Container
- * @param pipelineChannel - transport information for this container
- */
- public Pipeline(String containerName, PipelineChannel pipelineChannel) {
- this.containerName = containerName;
- this.pipelineChannel = pipelineChannel;
- data = null;
- }
-
- /**
- * Gets pipeline object from protobuf.
- *
- * @param pipeline - ProtoBuf definition for the pipeline.
- * @return Pipeline Object
- */
- public static Pipeline getFromProtoBuf(OzoneProtos.Pipeline pipeline) {
- Preconditions.checkNotNull(pipeline);
- PipelineChannel pipelineChannel =
- PipelineChannel.getFromProtoBuf(pipeline.getPipelineChannel());
- return new Pipeline(pipeline.getContainerName(), pipelineChannel);
- }
-
- public OzoneProtos.ReplicationFactor getFactor() {
- return pipelineChannel.getFactor();
- }
-
- /**
- * Returns the first machine in the set of datanodes.
- *
- * @return First Machine.
- */
- @JsonIgnore
- public DatanodeID getLeader() {
- return pipelineChannel.getDatanodes().get(pipelineChannel.getLeaderID());
- }
-
- /**
- * Returns the leader host.
- *
- * @return First Machine.
- */
- public String getLeaderHost() {
- return pipelineChannel.getDatanodes()
- .get(pipelineChannel.getLeaderID()).getHostName();
- }
-
- /**
- * Returns all machines that make up this pipeline.
- *
- * @return List of Machines.
- */
- @JsonIgnore
- public List<DatanodeID> getMachines() {
- return new ArrayList<>(pipelineChannel.getDatanodes().values());
- }
-
- /**
- * Returns all machines that make up this pipeline.
- *
- * @return List of Machines.
- */
- public List<String> getDatanodeHosts() {
- List<String> dataHosts = new ArrayList<>();
- for (DatanodeID id : pipelineChannel.getDatanodes().values()) {
- dataHosts.add(id.getHostName());
- }
- return dataHosts;
- }
-
- /**
- * Return a Protobuf Pipeline message from pipeline.
- *
- * @return Protobuf message
- */
- @JsonIgnore
- public OzoneProtos.Pipeline getProtobufMessage() {
- OzoneProtos.Pipeline.Builder builder =
- OzoneProtos.Pipeline.newBuilder();
- builder.setContainerName(this.containerName);
- builder.setPipelineChannel(this.pipelineChannel.getProtobufMessage());
- return builder.build();
- }
-
- /**
- * Returns containerName if available.
- *
- * @return String.
- */
- public String getContainerName() {
- return containerName;
- }
-
- /**
- * Returns private data that is set on this pipeline.
- *
- * @return blob, the user can interpret it any way they like.
- */
- public byte[] getData() {
- if (this.data != null) {
- return Arrays.copyOf(this.data, this.data.length);
- } else {
- return null;
- }
- }
-
- @VisibleForTesting
- public PipelineChannel getPipelineChannel() {
- return pipelineChannel;
- }
-
- /**
- * Set private data on pipeline.
- *
- * @param data -- private data.
- */
- public void setData(byte[] data) {
- if (data != null) {
- this.data = Arrays.copyOf(data, data.length);
- }
- }
-
- /**
- * Gets the State of the pipeline.
- *
- * @return - LifeCycleStates.
- */
- public OzoneProtos.LifeCycleState getLifeCycleState() {
- return pipelineChannel.getLifeCycleState();
- }
-
- /**
- * Gets the pipeline Name.
- *
- * @return - Name of the pipeline
- */
- public String getPipelineName() {
- return pipelineChannel.getName();
- }
-
- /**
- * Returns the type.
- *
- * @return type - Standalone, Ratis, Chained.
- */
- public OzoneProtos.ReplicationType getType() {
- return pipelineChannel.getType();
- }
-
- @Override
- public String toString() {
- final StringBuilder b = new StringBuilder(getClass().getSimpleName())
- .append("[");
- pipelineChannel.getDatanodes().keySet().stream()
- .forEach(id -> b.
- append(id.endsWith(pipelineChannel.getLeaderID()) ? "*" + id : id));
- b.append("] container:").append(containerName);
- b.append(" name:").append(getPipelineName());
- if (getType() != null) {
- b.append(" type:").append(getType().toString());
- }
- if (getFactor() != null) {
- b.append(" factor:").append(getFactor().toString());
- }
- if (getLifeCycleState() != null) {
- b.append(" State:").append(getLifeCycleState().toString());
- }
- return b.toString();
- }
-
- /**
- * Returns a JSON string of this object.
- *
- * @return String - json string
- * @throws IOException
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- @JsonFilter(PIPELINE_INFO)
- class MixIn {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
deleted file mode 100644
index 0af0534..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/PipelineChannel.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * PipelineChannel information for a {@link Pipeline}.
- */
-public class PipelineChannel {
- @JsonIgnore
- private String leaderID;
- @JsonIgnore
- private Map<String, DatanodeID> datanodes;
- private LifeCycleState lifeCycleState;
- private ReplicationType type;
- private ReplicationFactor factor;
- private String name;
-
- public PipelineChannel(String leaderID, LifeCycleState lifeCycleState,
- ReplicationType replicationType, ReplicationFactor replicationFactor,
- String name) {
- this.leaderID = leaderID;
- this.lifeCycleState = lifeCycleState;
- this.type = replicationType;
- this.factor = replicationFactor;
- this.name = name;
- datanodes = new TreeMap<>();
- }
-
- public String getLeaderID() {
- return leaderID;
- }
-
- public Map<String, DatanodeID> getDatanodes() {
- return datanodes;
- }
-
- public LifeCycleState getLifeCycleState() {
- return lifeCycleState;
- }
-
- public ReplicationType getType() {
- return type;
- }
-
- public ReplicationFactor getFactor() {
- return factor;
- }
-
- public String getName() {
- return name;
- }
-
- public void addMember(DatanodeID dataNodeId) {
- datanodes.put(dataNodeId.getDatanodeUuid(), dataNodeId);
- }
-
- @JsonIgnore
- public OzoneProtos.PipelineChannel getProtobufMessage() {
- OzoneProtos.PipelineChannel.Builder builder =
- OzoneProtos.PipelineChannel.newBuilder();
- for (DatanodeID datanode : datanodes.values()) {
- builder.addMembers(datanode.getProtoBufMessage());
- }
- builder.setLeaderID(leaderID);
-
- if (this.getLifeCycleState() != null) {
- builder.setState(this.getLifeCycleState());
- }
- if (this.getType() != null) {
- builder.setType(this.getType());
- }
-
- if (this.getFactor() != null) {
- builder.setFactor(this.getFactor());
- }
- return builder.build();
- }
-
- public static PipelineChannel getFromProtoBuf(
- OzoneProtos.PipelineChannel transportProtos) {
- Preconditions.checkNotNull(transportProtos);
- PipelineChannel pipelineChannel =
- new PipelineChannel(transportProtos.getLeaderID(),
- transportProtos.getState(),
- transportProtos.getType(),
- transportProtos.getFactor(),
- transportProtos.getName());
-
- for (HdfsProtos.DatanodeIDProto dataID : transportProtos.getMembersList()) {
- pipelineChannel.addMember(DatanodeID.getFromProtoBuf(dataID));
- }
- return pipelineChannel;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
deleted file mode 100644
index 549fced..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/StorageContainerException.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-
-import java.io.IOException;
-
-/**
- * Exceptions thrown from the Storage Container.
- */
-public class StorageContainerException extends IOException {
- private ContainerProtos.Result result;
-
- /**
- * Constructs an {@code IOException} with {@code null}
- * as its error detail message.
- */
- public StorageContainerException(ContainerProtos.Result result) {
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the {@link #getMessage()} method)
- * @param result - The result code
- */
- public StorageContainerException(String message,
- ContainerProtos.Result result) {
- super(message);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified detail message
- * and cause.
- * <p>
- * <p> Note that the detail message associated with {@code cause} is
- * <i>not</i> automatically incorporated into this exception's detail
- * message.
- *
- * @param message The detail message (which is saved for later retrieval by
- * the {@link #getMessage()} method)
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- *
- * @param result - The result code
- * @since 1.6
- */
- public StorageContainerException(String message, Throwable cause,
- ContainerProtos.Result result) {
- super(message, cause);
- this.result = result;
- }
-
- /**
- * Constructs an {@code IOException} with the specified cause and a
- * detail message of {@code (cause==null ? null : cause.toString())}
- * (which typically contains the class and detail message of {@code cause}).
- * This constructor is useful for IO exceptions that are little more
- * than wrappers for other throwables.
- *
- * @param cause The cause (which is saved for later retrieval by the {@link
- * #getCause()} method). (A null value is permitted, and indicates that the
- * cause is nonexistent or unknown.)
- * @param result - The result code
- * @since 1.6
- */
- public StorageContainerException(Throwable cause, ContainerProtos.Result
- result) {
- super(cause);
- this.result = result;
- }
-
- /**
- * Returns Result.
- *
- * @return Result.
- */
- public ContainerProtos.Result getResult() {
- return result;
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
deleted file mode 100644
index 3fa9663..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/container/common/helpers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index ad24f98..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-/**
- * This package contains classes for the client of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
deleted file mode 100644
index 469dab5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/LocatedContainer.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-import java.util.Set;
-
-/**
- * Holds the nodes that currently host the container for an object key hash.
- */
-@InterfaceAudience.Private
-public final class LocatedContainer {
- private final String key;
- private final String matchedKeyPrefix;
- private final String containerName;
- private final Set<DatanodeInfo> locations;
- private final DatanodeInfo leader;
-
- /**
- * Creates a LocatedContainer.
- *
- * @param key object key
- * @param matchedKeyPrefix prefix of key that was used to find the location
- * @param containerName container name
- * @param locations nodes that currently host the container
- * @param leader node that currently acts as pipeline leader
- */
- public LocatedContainer(String key, String matchedKeyPrefix,
- String containerName, Set<DatanodeInfo> locations, DatanodeInfo leader) {
- this.key = key;
- this.matchedKeyPrefix = matchedKeyPrefix;
- this.containerName = containerName;
- this.locations = locations;
- this.leader = leader;
- }
-
- /**
- * Returns the container name.
- *
- * @return container name
- */
- public String getContainerName() {
- return this.containerName;
- }
-
- /**
- * Returns the object key.
- *
- * @return object key
- */
- public String getKey() {
- return this.key;
- }
-
- /**
- * Returns the node that currently acts as pipeline leader.
- *
- * @return node that currently acts as pipeline leader
- */
- public DatanodeInfo getLeader() {
- return this.leader;
- }
-
- /**
- * Returns the nodes that currently host the container.
- *
- * @return Set<DatanodeInfo> nodes that currently host the container
- */
- public Set<DatanodeInfo> getLocations() {
- return this.locations;
- }
-
- /**
- * Returns the prefix of the key that was used to find the location.
- *
- * @return prefix of the key that was used to find the location
- */
- public String getMatchedKeyPrefix() {
- return this.matchedKeyPrefix;
- }
-
- @Override
- public boolean equals(Object otherObj) {
- if (otherObj == null) {
- return false;
- }
- if (!(otherObj instanceof LocatedContainer)) {
- return false;
- }
- LocatedContainer other = (LocatedContainer)otherObj;
- return this.key == null ? other.key == null : this.key.equals(other.key);
- }
-
- @Override
- public int hashCode() {
- return key.hashCode();
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName()
- + "{key=" + key
- + "; matchedKeyPrefix=" + matchedKeyPrefix
- + "; containerName=" + containerName
- + "; locations=" + locations
- + "; leader=" + leader
- + "}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
deleted file mode 100644
index 9728554..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmBlockLocationProtocol.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.scm.ScmInfo;
-
-/**
- * ScmBlockLocationProtocol is used by an HDFS node to find the set of nodes
- * to read/write a block.
- */
-public interface ScmBlockLocationProtocol {
-
- /**
- * Find the set of nodes to read/write a block, as
- * identified by the block key. This method supports batch lookup by
- * passing multiple keys.
- *
- * @param keys batch of block keys to find
- * @return allocated blocks for each block key
- * @throws IOException if there is any failure
- */
- Set<AllocatedBlock> getBlockLocations(Set<String> keys) throws IOException;
-
- /**
- * Asks SCM where a block should be allocated. SCM responds with the
- * set of datanodes that should be used creating this block.
- * @param size - size of the block.
- * @return allocated block accessing info (key, pipeline).
- * @throws IOException
- */
- AllocatedBlock allocateBlock(long size, ReplicationType type,
- ReplicationFactor factor, String owner) throws IOException;
-
- /**
- * Delete blocks for a set of object keys.
- *
- * @param keyBlocksInfoList Map of object key and its blocks.
- * @return list of block deletion results.
- * @throws IOException if there is any failure.
- */
- List<DeleteBlockGroupResult>
- deleteKeyBlocks(List<BlockGroup> keyBlocksInfoList) throws IOException;
-
- /**
- * Gets the Clusterid and SCM Id from SCM.
- */
- ScmInfo getScmInfo() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
deleted file mode 100644
index 4e4b3d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/ScmLocatedBlock.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-/**
- * Holds the nodes that currently host the block for a block key.
- */
-@InterfaceAudience.Private
-public final class ScmLocatedBlock {
- private final String key;
- private final List<DatanodeInfo> locations;
- private final DatanodeInfo leader;
-
- /**
- * Creates a ScmLocatedBlock.
- *
- * @param key object key
- * @param locations nodes that currently host the block
- * @param leader node that currently acts as pipeline leader
- */
- public ScmLocatedBlock(final String key, final List<DatanodeInfo> locations,
- final DatanodeInfo leader) {
- this.key = key;
- this.locations = locations;
- this.leader = leader;
- }
-
- /**
- * Returns the object key.
- *
- * @return object key
- */
- public String getKey() {
- return this.key;
- }
-
- /**
- * Returns the node that currently acts as pipeline leader.
- *
- * @return node that currently acts as pipeline leader
- */
- public DatanodeInfo getLeader() {
- return this.leader;
- }
-
- /**
- * Returns the nodes that currently host the block.
- *
- * @return List<DatanodeInfo> nodes that currently host the block
- */
- public List<DatanodeInfo> getLocations() {
- return this.locations;
- }
-
- @Override
- public boolean equals(Object otherObj) {
- if (otherObj == null) {
- return false;
- }
- if (!(otherObj instanceof ScmLocatedBlock)) {
- return false;
- }
- ScmLocatedBlock other = (ScmLocatedBlock)otherObj;
- return this.key == null ? other.key == null : this.key.equals(other.key);
- }
-
- @Override
- public int hashCode() {
- return key.hashCode();
- }
-
- @Override
- public String toString() {
- return getClass().getSimpleName() + "{key=" + key + "; locations="
- + locations.stream().map(loc -> loc.toString()).collect(Collectors
- .joining(",")) + "; leader=" + leader + "}";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
deleted file mode 100644
index ad78bd8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/StorageContainerLocationProtocol.java
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.scm.protocol;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-/**
- * ContainerLocationProtocol is used by an HDFS node to find the set of nodes
- * that currently host a container.
- */
-public interface StorageContainerLocationProtocol {
- /**
- * Asks SCM where a container should be allocated. SCM responds with the
- * set of datanodes that should be used creating this container.
- *
- */
- Pipeline allocateContainer(OzoneProtos.ReplicationType replicationType,
- OzoneProtos.ReplicationFactor factor, String containerName, String owner)
- throws IOException;
-
- /**
- * Ask SCM the location of the container. SCM responds with a group of
- * nodes where this container and its replicas are located.
- *
- * @param containerName - Name of the container.
- * @return Pipeline - the pipeline where container locates.
- * @throws IOException
- */
- Pipeline getContainer(String containerName) throws IOException;
-
- /**
- * Ask SCM a list of containers with a range of container names
- * and the limit of count.
- * Search container names between start name(exclusive), and
- * use prefix name to filter the result. the max size of the
- * searching range cannot exceed the value of count.
- *
- * @param startName start name, if null, start searching at the head.
- * @param prefixName prefix name, if null, then filter is disabled.
- * @param count count, if count < 0, the max size is unlimited.(
- * Usually the count will be replace with a very big
- * value instead of being unlimited in case the db is very big)
- *
- * @return a list of container.
- * @throws IOException
- */
- List<ContainerInfo> listContainer(String startName, String prefixName,
- int count) throws IOException;
-
- /**
- * Deletes a container in SCM.
- *
- * @param containerName
- * @throws IOException
- * if failed to delete the container mapping from db store
- * or container doesn't exist.
- */
- void deleteContainer(String containerName) throws IOException;
-
- /**
- * Queries a list of Node Statuses.
- * @param nodeStatuses
- * @return List of Datanodes.
- */
- OzoneProtos.NodePool queryNode(EnumSet<OzoneProtos.NodeState> nodeStatuses,
- OzoneProtos.QueryScope queryScope, String poolName) throws IOException;
-
- /**
- * Notify from client when begin or finish creating objects like pipeline
- * or containers on datanodes.
- * Container will be in Operational state after that.
- * @param type object type
- * @param name object name
- * @param op operation type (e.g., create, close, delete)
- * @param stage creation stage
- */
- void notifyObjectStageChange(
- ObjectStageChangeRequestProto.Type type, String name,
- ObjectStageChangeRequestProto.Op op,
- ObjectStageChangeRequestProto.Stage stage) throws IOException;
-
- /**
- * Creates a replication pipeline of a specified type.
- * @param type - replication type
- * @param factor - factor 1 or 3
- * @param nodePool - optional machine list to build a pipeline.
- * @throws IOException
- */
- Pipeline createReplicationPipeline(OzoneProtos.ReplicationType type,
- OzoneProtos.ReplicationFactor factor, OzoneProtos.NodePool nodePool)
- throws IOException;
-
- /**
- * Returns information about SCM.
- *
- * @return {@link ScmInfo}
- * @throws IOException
- */
- ScmInfo getScmInfo() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
deleted file mode 100644
index 274f859..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocol/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm.protocol;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 02d973a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,207 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Sets;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.ScmLocatedBlockProto;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.KeyBlocks;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link ScmBlockLocationProtocol} interface to the RPC server
- * implementing {@link ScmBlockLocationProtocolPB}.
- */
-@InterfaceAudience.Private
-public final class ScmBlockLocationProtocolClientSideTranslatorPB
- implements ScmBlockLocationProtocol, ProtocolTranslator, Closeable {
-
- /**
- * RpcController is not used and hence is set to null.
- */
- private static final RpcController NULL_RPC_CONTROLLER = null;
-
- private final ScmBlockLocationProtocolPB rpcProxy;
-
- /**
- * Creates a new StorageContainerLocationProtocolClientSideTranslatorPB.
- *
- * @param rpcProxy {@link StorageContainerLocationProtocolPB} RPC proxy
- */
- public ScmBlockLocationProtocolClientSideTranslatorPB(
- ScmBlockLocationProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- /**
- * Find the set of nodes to read/write a block, as
- * identified by the block key. This method supports batch lookup by
- * passing multiple keys.
- *
- * @param keys batch of block keys to find
- * @return allocated blocks for each block key
- * @throws IOException if there is any failure
- */
- @Override
- public Set<AllocatedBlock> getBlockLocations(Set<String> keys)
- throws IOException {
- GetScmBlockLocationsRequestProto.Builder req =
- GetScmBlockLocationsRequestProto.newBuilder();
- for (String key : keys) {
- req.addKeys(key);
- }
- final GetScmBlockLocationsResponseProto resp;
- try {
- resp = rpcProxy.getScmBlockLocations(NULL_RPC_CONTROLLER,
- req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- Set<AllocatedBlock> locatedBlocks =
- Sets.newLinkedHashSetWithExpectedSize(resp.getLocatedBlocksCount());
- for (ScmLocatedBlockProto locatedBlock : resp.getLocatedBlocksList()) {
- locatedBlocks.add(new AllocatedBlock.Builder()
- .setKey(locatedBlock.getKey())
- .setPipeline(Pipeline.getFromProtoBuf(locatedBlock.getPipeline()))
- .build());
- }
- return locatedBlocks;
- }
-
- /**
- * Asks SCM where a block should be allocated. SCM responds with the
- * set of datanodes that should be used creating this block.
- * @param size - size of the block.
- * @return allocated block accessing info (key, pipeline).
- * @throws IOException
- */
- @Override
- public AllocatedBlock allocateBlock(long size,
- OzoneProtos.ReplicationType type, OzoneProtos.ReplicationFactor factor,
- String owner) throws IOException {
- Preconditions.checkArgument(size > 0, "block size must be greater than 0");
-
- AllocateScmBlockRequestProto request =
- AllocateScmBlockRequestProto.newBuilder().setSize(size).setType(type)
- .setFactor(factor).setOwner(owner).build();
- final AllocateScmBlockResponseProto response;
- try {
- response = rpcProxy.allocateScmBlock(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- if (response.getErrorCode() !=
- AllocateScmBlockResponseProto.Error.success) {
- throw new IOException(response.hasErrorMessage() ?
- response.getErrorMessage() : "Allocate block failed.");
- }
- AllocatedBlock.Builder builder = new AllocatedBlock.Builder()
- .setKey(response.getKey())
- .setPipeline(Pipeline.getFromProtoBuf(response.getPipeline()))
- .setShouldCreateContainer(response.getCreateContainer());
- return builder.build();
- }
-
- /**
- * Delete the set of keys specified.
- *
- * @param keyBlocksInfoList batch of block keys to delete.
- * @return list of block deletion results.
- * @throws IOException if there is any failure.
- *
- */
- @Override
- public List<DeleteBlockGroupResult> deleteKeyBlocks(
- List<BlockGroup> keyBlocksInfoList) throws IOException {
- List<KeyBlocks> keyBlocksProto = keyBlocksInfoList.stream()
- .map(BlockGroup::getProto).collect(Collectors.toList());
- DeleteScmKeyBlocksRequestProto request = DeleteScmKeyBlocksRequestProto
- .newBuilder().addAllKeyBlocks(keyBlocksProto).build();
-
- final DeleteScmKeyBlocksResponseProto resp;
- try {
- resp = rpcProxy.deleteScmKeyBlocks(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- List<DeleteBlockGroupResult> results =
- new ArrayList<>(resp.getResultsCount());
- results.addAll(resp.getResultsList().stream().map(
- result -> new DeleteBlockGroupResult(result.getObjectKey(),
- DeleteBlockGroupResult
- .convertBlockResultProto(result.getBlockResultsList())))
- .collect(Collectors.toList()));
- return results;
- }
-
- /**
- * Gets the cluster Id and Scm Id from SCM.
- * @return ScmInfo
- * @throws IOException
- */
- @Override
- public ScmInfo getScmInfo() throws IOException {
- OzoneProtos.GetScmInfoRequestProto request =
- OzoneProtos.GetScmInfoRequestProto.getDefaultInstance();
- OzoneProtos.GetScmInfoRespsonseProto resp;
- try {
- resp = rpcProxy.getScmInfo(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- ScmInfo.Builder builder = new ScmInfo.Builder()
- .setClusterId(resp.getClusterId())
- .setScmId(resp.getScmId());
- return builder.build();
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public void close() {
- RPC.stopProxy(rpcProxy);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
deleted file mode 100644
index 256e735..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/protocolPB/ScmBlockLocationProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos
- .ScmBlockLocationProtocolService;
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager. This extends the
- * Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.ozone.protocol.ScmBlockLocationProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface ScmBlockLocationProtocolPB
- extends ScmBlockLocationProtocolService.BlockingInterface {
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
deleted file mode 100644
index 271ac74..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.css
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-*/
-body {
- padding: 40px;
- padding-top: 60px;
-}
-.starter-template {
- padding: 40px 15px;
- text-align: center;
-}
-
-
-.btn {
- border: 0 none;
- font-weight: 700;
- letter-spacing: 1px;
- text-transform: uppercase;
-}
-
-.btn:focus, .btn:active:focus, .btn.active:focus {
- outline: 0 none;
-}
-
-.table-striped > tbody > tr:nth-child(2n+1).selectedtag > td:hover {
- background-color: #3276b1;
-}
-.table-striped > tbody > tr:nth-child(2n+1).selectedtag > td {
- background-color: #3276b1;
-}
-.tagPanel tr.selectedtag td {
- background-color: #3276b1;
-}
-.top-buffer { margin-top:4px; }
-
-
-.sortorder:after {
- content: '\25b2'; // BLACK UP-POINTING TRIANGLE
-}
-.sortorder.reverse:after {
- content: '\25bc'; // BLACK DOWN-POINTING TRIANGLE
-}
-
-.wrap-table{
- word-wrap: break-word;
- table-layout: fixed;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
deleted file mode 100644
index 37cafef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/ozone.js
+++ /dev/null
@@ -1,355 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-(function () {
- "use strict";
-
- var isIgnoredJmxKeys = function (key) {
- return key == 'name' || key == 'modelerType' || key == "$$hashKey" ||
- key.match(/tag.*/);
- };
- angular.module('ozone', ['nvd3', 'ngRoute']);
- angular.module('ozone').config(function ($routeProvider) {
- $routeProvider
- .when("/", {
- templateUrl: "main.html"
- })
- .when("/metrics/rpc", {
- template: "<rpc-metrics></rpc-metrics>"
- })
- .when("/config", {
- template: "<config></config>"
- })
- });
- angular.module('ozone').component('overview', {
- templateUrl: 'static/templates/overview.html',
- transclude: true,
- controller: function ($http) {
- var ctrl = this;
- $http.get("jmx?qry=Hadoop:service=*,name=*,component=ServerRuntime")
- .then(function (result) {
- ctrl.jmx = result.data.beans[0]
- })
- }
- });
- angular.module('ozone').component('jvmParameters', {
- templateUrl: 'static/templates/jvm.html',
- controller: function ($http) {
- var ctrl = this;
- $http.get("jmx?qry=java.lang:type=Runtime")
- .then(function (result) {
- ctrl.jmx = result.data.beans[0];
-
- //convert array to a map
- var systemProperties = {};
- for (var idx in ctrl.jmx.SystemProperties) {
- var item = ctrl.jmx.SystemProperties[idx];
- systemProperties[item.key.replace(/\./g, "_")] = item.value;
- }
- ctrl.jmx.SystemProperties = systemProperties;
- })
- }
- });
-
- angular.module('ozone').component('rpcMetrics', {
- template: '<h1>Rpc metrics</h1><tabs>' +
- '<pane ng-repeat="metric in $ctrl.metrics" ' +
- 'title="{{metric[\'tag.serverName\']}} ({{metric[\'tag.port\']}})">' +
- '<rpc-metric jmxdata="metric"></rpc-metric></pane>' +
- '</tabs>',
- controller: function ($http) {
- var ctrl = this;
- $http.get("jmx?qry=Hadoop:service=*,name=RpcActivityForPort*")
- .then(function (result) {
- ctrl.metrics = result.data.beans;
- })
- }
- });
- angular.module('ozone').component('rpcMetric', {
- bindings: {
- jmxdata: '<'
- },
- templateUrl: 'static/templates/rpc-metrics.html',
- controller: function () {
- var ctrl = this;
-
-
- ctrl.percentileGraphOptions = {
- chart: {
- type: 'discreteBarChart',
- height: 450,
- margin: {
- top: 20,
- right: 20,
- bottom: 50,
- left: 55
- },
- x: function (d) {
- return d.label;
- },
- y: function (d) {
- return d.value;
- },
- showValues: true,
- valueFormat: function (d) {
- return d3.format(',.1f')(d);
- },
- duration: 500,
- xAxis: {
- axisLabel: 'Percentage'
- },
- yAxis: {
- axisLabel: 'Latency (ms)',
- axisLabelDistance: -10
- }
- }
- };
-
- ctrl.$onChanges = function (data) {
- var groupedMetrics = {}
-
- var createPercentageMetrics = function (metricName, window) {
- groupedMetrics.percentiles = groupedMetrics['percentiles'] || {}
- groupedMetrics.percentiles[window] = groupedMetrics.percentiles[window] || {};
- groupedMetrics.percentiles[window][metricName] = groupedMetrics.percentiles[window][metricName] || {
- graphdata: [{
- key: window,
- values: []
- }], numOps: 0
- };
-
- };
- var metrics = ctrl.jmxdata;
- for (var key in metrics) {
- var percentile = key.match(/(.*Time)(\d+s)(\d+th)PercentileLatency/);
- var percentileNumOps = key.match(/(.*Time)(\d+s)NumOps/);
- var successFailures = key.match(/(.*)(Success|Failures)/);
- var numAverages = key.match(/(.*Time)(NumOps|AvgTime)/);
- if (percentile) {
- var metricName = percentile[1];
- var window = percentile[2];
- var percentage = percentile[3]
- createPercentageMetrics(metricName, window);
-
-
- groupedMetrics.percentiles[window][metricName].graphdata[0]
- .values.push({
- label: percentage,
- value: metrics[key]
- })
- } else if (successFailures) {
- var metricName = successFailures[1];
- groupedMetrics.successfailures = groupedMetrics['successfailures'] || {}
- groupedMetrics.successfailures[metricName] = groupedMetrics.successfailures[metricName] || {
- success: 0,
- failures: 0
- };
- if (successFailures[2] == 'Success') {
- groupedMetrics.successfailures[metricName].success = metrics[key];
- } else {
- groupedMetrics.successfailures[metricName].failures = metrics[key];
- }
-
- } else if (numAverages) {
- var metricName = numAverages[1];
- groupedMetrics.numavgs = groupedMetrics['numavgs'] || {}
- groupedMetrics.numavgs[metricName] = groupedMetrics.numavgs[metricName] || {
- numOps: 0,
- avgTime: 0
- };
- if (numAverages[2] == 'NumOps') {
- groupedMetrics.numavgs[metricName].numOps = metrics[key];
- } else {
- groupedMetrics.numavgs[metricName].avgTime = metrics[key];
- }
-
- } else if (percentileNumOps) {
- var metricName = percentileNumOps[1];
- var window = percentileNumOps[2];
- createPercentageMetrics(metricName, window);
- groupedMetrics.percentiles[window][metricName].numOps = metrics[key];
- } else if (isIgnoredJmxKeys(key)) {
- //ignore
- } else {
- groupedMetrics.others = groupedMetrics.others || [];
- groupedMetrics.others.push({
- 'key': key,
- 'value': metrics[key]
- });
- }
-
- }
- ctrl.metrics = groupedMetrics;
- };
-
- }
- });
- angular.module('ozone')
- .component('tabs', {
- transclude: true,
- controller: function ($scope) {
- var ctrl = this;
- var panes = this.panes = [];
- this.select = function (pane) {
- angular.forEach(panes, function (pane) {
- pane.selected = false;
- });
- pane.selected = true;
- };
- this.addPane = function (pane) {
- if (panes.length === 0) {
- this.select(pane);
- }
- panes.push(pane);
- };
- this.click = function(pane) {
- ctrl.select(pane);
- }
- },
- template: '<div class="nav navtabs"><div class="row"><ul' +
- ' class="nav nav-pills">' +
- '<li ng-repeat="pane in $ctrl.panes" ng-class="{active:pane.selected}">' +
- '<a href="" ng-click="$ctrl.click(pane)">{{pane.title}}</a> ' +
- '</li> </ul></div><br/><div class="tab-content" ng-transclude></div> </div>'
- })
- .component('pane', {
- transclude: true,
- require: {
- tabsCtrl: '^tabs'
- },
- bindings: {
- title: '@'
- },
- controller: function () {
- this.$onInit = function () {
- this.tabsCtrl.addPane(this);
- };
- },
- template: '<div class="tab-pane" ng-if="$ctrl.selected" ng-transclude></div>'
- });
-
- angular.module('ozone').component('navmenu', {
- bindings: {
- metrics: '<'
- },
- templateUrl: 'static/templates/menu.html',
- controller: function ($http) {
- var ctrl = this;
- ctrl.docs = false;
- $http.head("docs/index.html")
- .then(function (result) {
- ctrl.docs = true;
- },function(){
- ctrl.docs = false;
- });
- }
- });
-
- angular.module('ozone').component('config', {
- templateUrl: 'static/templates/config.html',
- controller: function ($scope, $http) {
- var ctrl = this;
- ctrl.selectedTags = [];
-
- $http.get("conf?cmd=getOzoneTags&group=ozone")
- .then(function (response) {
- ctrl.tags = response.data;
-
- var excludedTags = ['CBLOCK', 'KSM', 'SCM'];
- for (var i = 0; i < excludedTags.length; i++) {
- var idx = ctrl.tags.indexOf(excludedTags[i]);
- // Remove CBLOCK related properties
- if (idx > -1) {
- ctrl.tags.splice(idx, 1);
- }
- }
- ctrl.loadAll();
- });
-
-
-
- ctrl.loadAll = function () {
- console.log("Displaying all configs");
- $http.get("conf?cmd=getPropertyByTag&tags=" + ctrl.tags + "&group=ozone").then(function (response) {
- ctrl.configs = response.data;
- console.log(ctrl.configs)
- for (var idx in ctrl.configs) {
- var tags = []
- var parsedTags = ctrl.configs[idx].tag.split(",");
- for (var t in parsedTags) {
- tags.push(parsedTags[t].trim())
- }
- ctrl.configs[idx].tag = tags;
-
- };
- ctrl.sortBy('name');
- });
- };
-
- ctrl.tagFilter = function (value, index, array) {
- if (!ctrl.selectedTags) {
- return true;
- }
- var selected = true;
- for (var idx in ctrl.selectedTags) {
- selected = selected && (value.tag.indexOf(ctrl.selectedTags[idx]) > -1);
- }
- return selected;
- };
- ctrl.configFilter = function (config) {
- return false;
- };
- ctrl.selected = function (tag) {
- return ctrl.selectedTags.includes(tag);
- };
-
- ctrl.allSelected = function () {
- return ctrl.selectedTags.indexOf('SCM') == -1
- && ctrl.selectedTags.indexOf('KSM') == -1
- };
-
- ctrl.switchto = function (tag) {
- var tags = ctrl.selectedTags.filter(function (item) {
- return item != 'KSM' && item != 'SCM';
- });
- if (tag) {
- tags.push(tag);
- }
- ctrl.selectedTags = tags;
- };
-
- ctrl.select = function (tag) {
- var tagIdx = ctrl.selectedTags.indexOf(tag);
- if (tagIdx > -1) {
- ctrl.selectedTags = ctrl.selectedTags.filter(function (item) {
- return item != tag;
- });
- } else {
- ctrl.selectedTags.push(tag);
- }
- console.log("Tags selected:" + ctrl.selectedTags);
- };
-
- ctrl.sortBy = function (propertyName) {
- ctrl.reverse = (ctrl.propertyName === propertyName) ? !ctrl.reverse : false;
- ctrl.propertyName = propertyName;
- };
-
- }
- });
-
-})();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
deleted file mode 100644
index 0a273c3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/config.html
+++ /dev/null
@@ -1,92 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
-
-<div class="row top-buffer">
- <div class="col-md-2">
- <input type="text" class="form-control" placeholder="Search Properties"
- name="search" ng-model="search.$">
- </div>
- <div class="col-md-10">
- <div class="btn-group btn-group-justified">
- <a class="btn"
- ng-class="$ctrl.allSelected() ? 'btn-primary' :'btn-secondary'"
- ng-click="$ctrl.switchto('')">All
- </a>
- <a class="btn"
- ng-class="$ctrl.selected('KSM') ? 'btn-primary' :'btn-secondary'"
- ng-click="$ctrl.switchto('KSM')">KSM</a>
- <a class="btn"
- ng-class="$ctrl.selected('SCM') ? 'btn-primary' :'btn-secondary'"
- ng-click="$ctrl.switchto('SCM')">SCM</a>
- </div>
- </div>
-</div>
-<div class="row">
- <div class="col-md-2">
-
- <table class="table table-striped table-condensed tagPanel">
- <colgroup>
- <col class="col-md-12">
- </colgroup>
- <thead>
- <tr>
- <th>Tag</th>
- </tr>
- </thead>
- <tbody>
- <tr ng-click="$ctrl.select(tag)"
- ng-class="$ctrl.selected(tag) ? 'selectedtag':''"
- ng-repeat="tag in $ctrl.tags">
- <td>{{tag}}</td>
- </tr>
- </tbody>
- </table>
- </div>
- <div class="col-md-10">
- <table class="table table-striped table-condensed table-hover wrap-table">
- <thead>
- <tr>
- <th class="col-md-3" >
- <a href="#" ng-click="$ctrl.sortBy('name')">Property</a>
- <span class="sortorder" ng-show="propertyName === 'name'"
- ng-class="{reverse: reverse}">
-
- </span>
- </th>
- <th class="col-md-2" style="word-wrap: break-word;">
- <a ng-click="$ctrl.sortBy('value')">Value</a>
- <span class="sortorder" ng-show="propertyName === 'value'"
- ng-class="{reverse: reverse}"></span>
- </th>
- <th class="col-md-7">
- <a href="#" ng-click="$ctrl.sortBy('description')">Description</a>
- <span class="sortorder" ng-show="propertyName === 'description'"
- ng-class="{reverse: reverse}"></span>
- </th>
- </tr>
- </thead>
- <tbody>
- <tr
- ng-repeat="config in $ctrl.configs | filter:$ctrl.tagFilter | filter:search | orderBy:propertyName:reverse">
- <td style="word-wrap: break-word;">{{config.name}}</td>
- <td style="word-wrap: break-word;">{{config.value}}</td>
- <td style="word-wrap: break-word;">{{config.description}}</td>
- </tr>
- </tbody>
- </table>
- </div>
-</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
deleted file mode 100644
index c1f7d16..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/jvm.html
+++ /dev/null
@@ -1,26 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<table class="table table-bordered table-striped">
- <tr>
- <th>JVM:</th>
- <td>{{$ctrl.jmx.SystemProperties.java_vm_name}} {{$ctrl.jmx.SystemProperties.java_vm_version}}</td>
- </tr>
- <tr>
- <th>Input arguments:</th>
- <td>{{$ctrl.jmx.InputArguments}}</td>
- </tr>
-</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
deleted file mode 100644
index 95f1b48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/menu.html
+++ /dev/null
@@ -1,60 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<div id="navbar" class="collapse navbar-collapse">
- <ul class="nav navbar-nav" id="ui-tabs">
- <li>
- <a class="dropdown-toggle"
- id="metricsMenu"
- data-toggle="dropdown"
- aria-haspopup="true"
- aria-expanded="true">
- Metrics
- <span class="caret"></span>
- </a>
- <ul
- class="dropdown-menu"
- aria-labelledby="metricsMenu">
- <li ng-repeat="(name, url) in $ctrl.metrics">
- <a ng-href="{{url}}">{{name}}<span
- aria-hidden="true"></span></a></li>
- </ul>
- </li>
- <li><a href="#!/config">Configuration</a></li>
- <li ng-show="$ctrl.docs"><a href="/docs">Documentation</a></li>
- <li>
- <a class="dropdown-toggle"
- id="toolsMenu"
- data-toggle="dropdown"
- aria-haspopup="true"
- aria-expanded="true"
- >
- Common tools
- <span class="caret"></span>
- </a>
- <ul class="dropdown-menu" aria-labelledby="toolsMenu">
- <li><a href="jmx">JMX <span
- aria-hidden="true"></span></a></li>
- <li><a href="conf">Config <span
- aria-hidden="true"></a></li>
- <li><a href="stacks">Stacks <span
- aria-hidden="true"></a></li>
- <li><a href="logLevel">Log levels <span
- aria-hidden="true"></a></li>
- </ul>
- </li>
- </ul>
-</div><!--/.nav-collapse -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
deleted file mode 100644
index 30e2d26..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/overview.html
+++ /dev/null
@@ -1,39 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<h1>Overview</h1>
-<table class="table table-bordered table-striped">
- <tbody>
- <tr>
- <th>Started:</th>
- <td>{{$ctrl.jmx.StartedTimeInMillis | date : 'medium'}}</td>
- </tr>
- <tr>
- <th>Version:</th>
- <td>{{$ctrl.jmx.Version}}</td>
- </tr>
- <tr>
- <th>Compiled:</th>
- <td>{{$ctrl.jmx.CompileInfo}}</td>
- </tr>
- </tbody>
-</table>
-
-<h2>JVM parameters</h2>
-
-<jvm-parameters></jvm-parameters>
-
-<div ng-transclude></div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
deleted file mode 100644
index facb152..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/templates/rpc-metrics.html
+++ /dev/null
@@ -1,87 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<div ng-hide="$ctrl.metrics.percentiles" class="alert alert-info">
- Please set <b>rpc.metrics.quantile.enable</b> to <b>true</b> and define the
- intervals in seconds with setting <b>rpc.metrics.percentiles.intervals</b>
- (eg. set to <b>60,300</b>) in your hdfs-site.xml
- to display Hadoop RPC related graphs.
-</div>
-<div ng-repeat="(window,windowed) in $ctrl.metrics.percentiles">
- <h2>{{window}} window</h2>
- <p>Quantiles based on a fixed {{window}} window. Calculated once at every
- {{window}}</p>
-
- <div class="row">
- <div class="col-md-6 col-lg-4"
- ng-repeat="(metric,percentiles) in windowed">
- <h3>{{metric}}</h3>
- <p>{{percentiles.numOps}} sample</p>
- <nvd3 options="$ctrl.percentileGraphOptions"
- data="percentiles.graphdata"></nvd3>
- </div>
- </div>
-
-</div>
-<div class="row">
- <div ng-show="$ctrl.metrics.numavgs" class="col-md-6">
- <h2>Number of ops / Averages</h2>
-
- <table class="table table-bordered table-striped">
- <thead>
- <tr>
- <th>Metric name</th>
- <th>Number of ops</th>
- <th>Average time (ms)</th>
- </tr>
- </thead>
- <tr ng-repeat="(key,metric) in $ctrl.metrics.numavgs">
- <td>{{key}}</td>
- <td>{{metric.numOps | number}}</td>
- <td>{{metric.avgTime | number:2}}</td>
- </tr>
- </table>
- </div>
- <div ng-show="$ctrl.metrics.successfailures" class="col-md-6">
- <h2>Success / Failures</h2>
-
- <table class="table table-bordered table-striped">
- <thead>
- <tr>
- <th>Metric name</th>
- <th>Success</th>
- <th>Failures</th>
- </tr>
- </thead>
-
- <tr ng-repeat="(key,metric) in $ctrl.metrics.successfailures">
- <td>{{key}}</td>
- <td>{{metric.success}}</td>
- <td>{{metric.failures}}</td>
- </tr>
- </table>
- </div>
-</div>
-<div ng-show="$ctrl.metrics.others">
- <h2>Other JMX Metrics</h2>
-
- <table class="table">
- <tr ng-repeat="metric in $ctrl.metrics.others">
- <td>{{metric.key}}</td>
- <td>{{metric.value}}</td>
- </tr>
- </table>
-</div>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
deleted file mode 100644
index 9df974f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneCommandShell.md
+++ /dev/null
@@ -1,150 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-Ozone Command Shell
-===================
-
-Ozone command shell gives a command shell interface to work against ozone.
-Please note that this document assumes that cluster is deployed
-with simple authentication.
-
-The Ozone commands take the following format.
-
-* `hdfs oz --command_ http://hostname:port/volume/bucket/key -user
-<name> -root`
-
-The *port* specified in command should match the port mentioned in the config
-property `dfs.datanode.http.address`. This property can be set in `hdfs-site.xml`.
-The default value for the port is `9864` and is used in below commands.
-
-The *--root* option is a command line short cut that allows *hdfs oz*
-commands to be run as the user that started the cluster. This is useful to
-indicate that you want the commands to be run as some admin user. The only
-reason for this option is that it makes the life of a lazy developer more
-easier.
-
-Ozone Volume Commands
---------------------
-
-The volume commands allow users to create, delete and list the volumes in the
-ozone cluster.
-
-### Create Volume
-
-Volumes can be created only by Admins. Here is an example of creating a volume.
-
-* `hdfs oz -createVolume http://localhost:9864/hive -user bilbo -quota
-100TB -root`
-
-The above command creates a volume called `hive` owned by user `bilbo`. The
-`--root` option allows the command to be executed as user `hdfs` which is an
-admin in the cluster.
-
-### Update Volume
-
-Updates information like ownership and quota on an existing volume.
-
-* `hdfs oz -updateVolume http://localhost:9864/hive -quota 500TB -root`
-
-The above command changes the volume quota of hive from 100TB to 500TB.
-
-### Delete Volume
-Deletes a Volume if it is empty.
-
-* `hdfs oz -deleteVolume http://localhost:9864/hive -root`
-
-
-### Info Volume
-Info volume command allows the owner or the administrator of the cluster to read meta-data about a specific volume.
-
-* `hdfs oz -infoVolume http://localhost:9864/hive -root`
-
-### List Volumes
-
-List volume command can be used by administrator to list volumes of any user. It can also be used by a user to list volumes owned by him.
-
-* `hdfs oz -listVolume http://localhost:9864/ -user bilbo -root`
-
-The above command lists all volumes owned by user bilbo.
-
-Ozone Bucket Commands
---------------------
-
-Bucket commands follow a similar pattern as volume commands. However bucket commands are designed to be run by the owner of the volume.
-Following examples assume that these commands are run by the owner of the volume or bucket.
-
-
-### Create Bucket
-
-Create bucket call allows the owner of a volume to create a bucket.
-
-* `hdfs oz -createBucket http://localhost:9864/hive/january`
-
-This call creates a bucket called `january` in the volume called `hive`. If
-the volume does not exist, then this call will fail.
-
-
-### Update Bucket
-Updates bucket meta-data, like ACLs.
-
-* `hdfs oz -updateBucket http://localhost:9864/hive/january -addAcl
-user:spark:rw`
-
-### Delete Bucket
-Deletes a bucket if it is empty.
-
-* `hdfs oz -deleteBucket http://localhost:9864/hive/january`
-
-### Info Bucket
-Returns information about a given bucket.
-
-* `hdfs oz -infoBucket http://localhost:9864/hive/january`
-
-### List Buckets
-List buckets on a given volume.
-
-* `hdfs oz -listBucket http://localhost:9864/hive`
-
-Ozone Key Commands
-------------------
-
-Ozone key commands allows users to put, delete and get keys from ozone buckets.
-
-### Put Key
-Creates or overwrites a key in ozone store, -file points to the file you want
-to upload.
-
-* `hdfs oz -putKey http://localhost:9864/hive/january/processed.orc -file
-processed.orc`
-
-### Get Key
-Downloads a file from the ozone bucket.
-
-* `hdfs oz -getKey http://localhost:9864/hive/january/processed.orc -file
- processed.orc.copy`
-
-### Delete Key
-Deletes a key from the ozone store.
-
-* `hdfs oz -deleteKey http://localhost:9864/hive/january/processed.orc`
-
-### Info Key
-Reads key metadata from the ozone store.
-
-* `hdfs oz -infoKey http://localhost:9864/hive/january/processed.orc`
-
-### List Keys
-List all keys in an ozone bucket.
-
-* `hdfs oz -listKey http://localhost:9864/hive/january`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
deleted file mode 100644
index 3395a8e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneGettingStarted.md.vm
+++ /dev/null
@@ -1,320 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-Ozone - Object store for Hadoop
-==============================
-
-Introduction
-------------
-Ozone is an object store for Hadoop. It is a redundant, distributed object
-store build by leveraging primitives present in HDFS. Ozone supports REST
-API for accessing the store.
-
-Getting Started
----------------
-Ozone is a work in progress and currently lives in its own branch. To
-use it, you have to build a package by yourself and deploy a cluster.
-
-### Building Ozone
-
-To build Ozone, please checkout the hadoop sources from github. Then
-checkout the ozone branch, HDFS-7240 and build it.
-
-- `git checkout HDFS-7240`
-- `mvn clean package -DskipTests=true -Dmaven.javadoc.skip=true -Pdist -Dtar -DskipShade`
-
-skipShade is just to make compilation faster and not really required.
-
-This will give you a tarball in your distribution directory. This is the
-tarball that can be used for deploying your hadoop cluster. Here is an
-example of the tarball that will be generated.
-
-* `~/apache/hadoop/hadoop-dist/target/${project.version}.tar.gz`
-
-At this point we have an option to setup a physical cluster or run ozone via
-docker.
-
-Running Ozone via Docker
-------------------------
-
-This assumes that you have a running docker setup on the machine. Please run
-these following commands to see ozone in action.
-
- Go to the directory where the docker compose files exist.
-
-
- - `cd dev-support/compose/ozone`
-
-Tell docker to start ozone, this will start a KSM, SCM and a single datanode in
-the background.
-
-
- - `docker-compose up -d`
-
-Now let us run some work load against ozone, to do that we will run freon.
-
-This will log into the datanode and run bash.
-
- - `docker-compose exec datanode bash`
- - `cd hadoop/bin`
-
-Now you can run the oz command shell or freon, the ozone load generator.
-
-This is the command to run freon.
-
- - `./hdfs freon -mode offline -validateWrites -numOfVolumes 1 -numOfBuckets 10 -numOfKeys 100`
-
-You can checkout the KSM UI to see the requests information.
-
- - `http://localhost:9874/`
-
-If you need more datanode you can scale up:
-
- - `docker-compose scale datanode=3`
-
-Running Ozone using a real cluster
-----------------------------------
-
-Please proceed to setup a hadoop cluster by creating the hdfs-site.xml and
-other configuration files that are needed for your cluster.
-
-
-### Ozone Configuration
-
-Ozone relies on its own configuration file called `ozone-site.xml`. It is
-just for convenience and ease of management -- you can add these settings
-to `hdfs-site.xml`, if you don't want to keep ozone settings separate.
-This document refers to `ozone-site.xml` so that ozone settings are in one
-place and not mingled with HDFS settings.
-
- * _*ozone.enabled*_ This is the most important setting for ozone.
- Currently, Ozone is an opt-in subsystem of HDFS. By default, Ozone is
- disabled. Setting this flag to `true` enables ozone in the HDFS cluster.
- Here is an example,
-
-```
- <property>
- <name>ozone.enabled</name>
- <value>True</value>
- </property>
-```
- * _*ozone.metadata.dirs*_ Ozone is designed with modern hardware
- in mind. It tries to use SSDs effectively. So users can specify where the
- metadata must reside. Usually you pick your fastest disk (SSD if
- you have them on your nodes). KSM, SCM and datanode will write the metadata
- to these disks. This is a required setting, if this is missing Ozone will
- fail to come up. Here is an example,
-
-```
- <property>
- <name>ozone.metadata.dirs</name>
- <value>/data/disk1/meta</value>
- </property>
-```
-
-* _*ozone.scm.names*_ Ozone is build on top of container framework (See Ozone
- Architecture TODO). Storage container manager(SCM) is a distributed block
- service which is used by ozone and other storage services.
- This property allows datanodes to discover where SCM is, so that
- datanodes can send heartbeat to SCM. SCM is designed to be highly available
- and datanodes assume there are multiple instances of SCM which form a highly
- available ring. The HA feature of SCM is a work in progress. So we
- configure ozone.scm.names to be a single machine. Here is an example,
-
-```
- <property>
- <name>ozone.scm.names</name>
- <value>scm.hadoop.apache.org</value>
- </property>
-```
-
-* _*ozone.scm.datanode.id*_ Each datanode that speaks to SCM generates an ID
-just like HDFS. This is an optional setting. Please note:
-This path will be created by datanodes if it doesn't exist already. Here is an
- example,
-
-```
- <property>
- <name>ozone.scm.datanode.id</name>
- <value>/data/disk1/scm/meta/node/datanode.id</value>
- </property>
-```
-
-* _*ozone.scm.block.client.address*_ Storage Container Manager(SCM) offers a
- set of services that can be used to build a distributed storage system. One
- of the services offered is the block services. KSM and HDFS would use this
- service. This property describes where KSM can discover SCM's block service
- endpoint. There is corresponding ports etc, but assuming that we are using
- default ports, the server address is the only required field. Here is an
- example,
-
-```
- <property>
- <name>ozone.scm.block.client.address</name>
- <value>scm.hadoop.apache.org</value>
- </property>
-```
-
-* _*ozone.ksm.address*_ KSM server address. This is used by Ozonehandler and
-Ozone File System.
-
-```
- <property>
- <name>ozone.ksm.address</name>
- <value>ksm.hadoop.apache.org</value>
- </property>
-```
-
-Here is a quick summary of settings needed by Ozone.
-
-| Setting | Value | Comment |
-|--------------------------------|------------------------------|------------------------------------------------------------------|
-| ozone.enabled | True | This enables SCM and containers in HDFS cluster. |
-| ozone.metadata.dirs | file path | The metadata will be stored here. |
-| ozone.scm.names | SCM server name | Hostname:port or or IP:port address of SCM. |
-| ozone.scm.block.client.address | SCM server name and port | Used by services like KSM |
-| ozone.scm.client.address | SCM server name and port | Used by client side |
-| ozone.scm.datanode.address | SCM server name and port | Used by datanode to talk to SCM |
-| ozone.ksm.address | KSM server name | Used by Ozone handler and Ozone file system. |
-
- Here is a working example of`ozone-site.xml`.
-
-```
- <?xml version="1.0" encoding="UTF-8"?>
- <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
- <configuration>
- <property>
- <name>ozone.enabled</name>
- <value>True</value>
- </property>
-
- <property>
- <name>ozone.metadata.dirs</name>
- <value>/data/disk1/ozone/meta</value>
- </property>
-
- <property>
- <name>ozone.scm.names</name>
- <value>127.0.0.1</value>
- </property>
-
- <property>
- <name>ozone.scm.client.address</name>
- <value>127.0.0.1:9860</value>
- </property>
-
- <property>
- <name>ozone.scm.block.client.address</name>
- <value>127.0.0.1:9863</value>
- </property>
-
- <property>
- <name>ozone.scm.datanode.address</name>
- <value>127.0.0.1:9861</value>
- </property>
-
- <property>
- <name>ozone.ksm.address</name>
- <value>127.0.0.1:9874</value>
- </property>
- </configuration>
-```
-
-### Starting Ozone
-
-Ozone is designed to run concurrently with HDFS. The simplest way to [start
-HDFS](../hadoop-common/ClusterSetup.html) is to run `start-dfs.sh` from the
-`$HADOOP/sbin/start-dfs.sh`. Once HDFS
-is running, please verify it is fully functional by running some commands like
-
- - *./hdfs dfs -mkdir /usr*
- - *./hdfs dfs -ls /*
-
- Once you are sure that HDFS is running, start Ozone. To start ozone, you
- need to start SCM and KSM. Currently we assume that both KSM and SCM
- is running on the same node, this will change in future.
-
- The first time you bring up Ozone, SCM must be initialized.
-
- - `./hdfs scm -init`
-
- Start SCM.
-
- - `./hdfs --daemon start scm`
-
- Once SCM gets started, KSM must be initialized.
-
- - `./hdfs ksm -createObjectStore`
-
- Start KSM.
-
- - `./hdfs --daemon start ksm`
-
-if you would like to start HDFS and Ozone together, you can do that by running
- a single command.
- - `$HADOOP/sbin/start-ozone.sh`
-
- This command will start HDFS and then start the ozone components.
-
- Once you have ozone running you can use these ozone [shell](./OzoneCommandShell.html)
- commands to create a volume, bucket and keys.
-
-### Diagnosing issues
-
-Ozone tries not to pollute the existing HDFS streams of configuration and
-logging. So ozone logs are by default configured to be written to a file
-called `ozone.log`. This is controlled by the settings in `log4j.properties`
-file in the hadoop configuration directory.
-
-Here is the log4j properties that are added by ozone.
-
-
-```
- #
- # Add a logger for ozone that is separate from the Datanode.
- #
- #log4j.debug=true
- log4j.logger.org.apache.hadoop.ozone=DEBUG,OZONE,FILE
-
- # Do not log into datanode logs. Remove this line to have single log.
- log4j.additivity.org.apache.hadoop.ozone=false
-
- # For development purposes, log both to console and log file.
- log4j.appender.OZONE=org.apache.log4j.ConsoleAppender
- log4j.appender.OZONE.Threshold=info
- log4j.appender.OZONE.layout=org.apache.log4j.PatternLayout
- log4j.appender.OZONE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
- %X{component} %X{function} %X{resource} %X{user} %X{request} - %m%n
-
- # Real ozone logger that writes to ozone.log
- log4j.appender.FILE=org.apache.log4j.DailyRollingFileAppender
- log4j.appender.FILE.File=${hadoop.log.dir}/ozone.log
- log4j.appender.FILE.Threshold=debug
- log4j.appender.FILE.layout=org.apache.log4j.PatternLayout
- log4j.appender.FILE.layout.ConversionPattern=%d{ISO8601} [%t] %-5p \
- (%F:%L) %X{function} %X{resource} %X{user} %X{request} - \
- %m%n
-```
-
-If you would like to have a single datanode log instead of ozone stuff
-getting written to ozone.log, please remove this line or set this to true.
-
- ` log4j.additivity.org.apache.hadoop.ozone=false`
-
-On the SCM/KSM side, you will be able to see
-
- - `hadoop-hdfs-ksm-hostname.log`
- - `hadoop-hdfs-scm-hostname.log`
-
-Please file any issues you see under [Object store in HDFS (HDFS-7240)](https://issues.apache.org/jira/browse/HDFS-7240)
-as this is still a work in progress.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
deleted file mode 100644
index f5eccf6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneMetrics.md
+++ /dev/null
@@ -1,166 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-
-
-HDFS Ozone Metrics
-===============
-
-<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-
-Overview
---------
-
-The container metrics that is used in HDFS Ozone.
-
-### Storage Container Metrics
-
-The metrics for various storage container operations in HDFS Ozone.
-
-Storage container is an optional service that can be enabled by setting
-'ozone.enabled' to true.
-These metrics are only available when ozone is enabled.
-
-Storage Container Metrics maintains a set of generic metrics for all
-container RPC calls that can be made to a datandoe/container.
-
-Along with the total number of RPC calls containers maintain a set of metrics
-for each RPC call. Following is the set of counters maintained for each RPC
-operation.
-
-*Total number of operation* - We maintain an array which counts how
-many times a specific operation has been performed.
-Eg.`NumCreateContainer` tells us how many times create container has been
-invoked on this datanode.
-
-*Total number of pending operation* - This is an array which counts how
-many times a specific operation is waitting to be processed from the client
-point of view.
-Eg.`NumPendingCreateContainer` tells us how many create container requests that
-waitting to be processed.
-
-*Average latency of each pending operation in nanoseconds* - The average latency
-of the operation from the client point of view.
-Eg. `CreateContainerLatencyAvgTime` - This tells us the average latency of
-Create Container from the client point of view.
-
-*Number of bytes involved in a specific command* - This is an array that is
-maintained for all operations, but makes sense only for read and write
-operations.
-
-While it is possible to read the bytes in update container, it really makes
-no sense, since no data stream involved. Users are advised to use this
-metric only when it makes sense. Eg. `BytesReadChunk` -- Tells us how
-many bytes have been read from this data using Read Chunk operation.
-
-*Average Latency of each operation* - The average latency of the operation.
-Eg. `LatencyCreateContainerAvgTime` - This tells us the average latency of
-Create Container.
-
-*Quantiles for each of these operations* - The 50/75/90/95/99th percentile
-of these operations. Eg. `CreateContainerNanos60s50thPercentileLatency` --
-gives latency of the create container operations at the 50th percentile latency
-(1 minute granularity). We report 50th, 75th, 90th, 95th and 99th percentile
-for all RPCs.
-
-So this leads to the containers reporting these counters for each of these
-RPC operations.
-
-| Name | Description |
-|:---- |:---- |
-| `NumOps` | Total number of container operations |
-| `CreateContainer` | Create container operation |
-| `ReadContainer` | Read container operation |
-| `UpdateContainer` | Update container operations |
-| `DeleteContainer` | Delete container operations |
-| `ListContainer` | List container operations |
-| `PutKey` | Put key operations |
-| `GetKey` | Get key operations |
-| `DeleteKey` | Delete key operations |
-| `ListKey` | List key operations |
-| `ReadChunk` | Read chunk operations |
-| `DeleteChunk` | Delete chunk operations |
-| `WriteChunk` | Write chunk operations|
-| `ListChunk` | List chunk operations |
-| `CompactChunk` | Compact chunk operations |
-| `PutSmallFile` | Put small file operations |
-| `GetSmallFile` | Get small file operations |
-| `CloseContainer` | Close container operations |
-
-### Storage Container Manager Metrics
-
-The metrics for containers that managed by Storage Container Manager.
-
-Storage Container Manager (SCM) is a master service which keeps track of
-replicas of storage containers. It also manages all data nodes and their
-states, dealing with container reports and dispatching commands for execution.
-
-Following are the counters for containers:
-
-| Name | Description |
-|:---- |:---- |
-| `LastContainerReportSize` | Total size in bytes of all containers in latest container report that SCM received from datanode |
-| `LastContainerReportUsed` | Total number of bytes used by all containers in latest container report that SCM received from datanode |
-| `LastContainerReportKeyCount` | Total number of keys in all containers in latest container report that SCM received from datanode |
-| `LastContainerReportReadBytes` | Total number of bytes have been read from all containers in latest container report that SCM received from datanode |
-| `LastContainerReportWriteBytes` | Total number of bytes have been written into all containers in latest container report that SCM received from datanode |
-| `LastContainerReportReadCount` | Total number of times containers have been read from in latest container report that SCM received from datanode |
-| `LastContainerReportWriteCount` | Total number of times containers have been written to in latest container report that SCM received from datanode |
-| `ContainerReportSize` | Total size in bytes of all containers over whole cluster |
-| `ContainerReportUsed` | Total number of bytes used by all containers over whole cluster |
-| `ContainerReportKeyCount` | Total number of keys in all containers over whole cluster |
-| `ContainerReportReadBytes` | Total number of bytes have been read from all containers over whole cluster |
-| `ContainerReportWriteBytes` | Total number of bytes have been written into all containers over whole cluster |
-| `ContainerReportReadCount` | Total number of times containers have been read from over whole cluster |
-| `ContainerReportWriteCount` | Total number of times containers have been written to over whole cluster |
-
-### Key Space Metrics
-
-The metrics for various key space manager operations in HDFS Ozone.
-
-key space manager (KSM) is a service that similar to the Namenode in HDFS.
-In the current design of KSM, it maintains metadata of all volumes, buckets and keys.
-These metrics are only available when ozone is enabled.
-
-Following is the set of counters maintained for each key space operation.
-
-*Total number of operation* - We maintain an array which counts how
-many times a specific operation has been performed.
-Eg.`NumVolumeCreate` tells us how many times create volume has been
-invoked in KSM.
-
-*Total number of failed operation* - This type operation is opposite to the above
-operation.
-Eg.`NumVolumeCreateFails` tells us how many times create volume has been invoked
-failed in KSM.
-
-Following are the counters for each of key space operations.
-
-| Name | Description |
-|:---- |:---- |
-| `VolumeCreate` | Create volume operation |
-| `VolumeUpdates` | Update volume property operation |
-| `VolumeInfos` | Get volume information operation |
-| `VolumeCheckAccesses` | Check volume access operation |
-| `VolumeDeletes` | Delete volume operation |
-| `VolumeLists` | List volume operation |
-| `BucketCreates` | Create bucket operation |
-| `BucketInfos` | Get bucket information operation |
-| `BucketUpdates` | Update bucket property operation |
-| `BucketDeletes` | Delete bucket operation |
-| `BucketLists` | List bucket operation |
-| `KeyAllocate` | Allocate key operation |
-| `KeyLookup` | Look up key operation |
-| `KeyDeletes` | Delete key operation |
-| `KeyLists` | List key operation |
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
deleted file mode 100644
index 41d7dbd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneOverview.md
+++ /dev/null
@@ -1,88 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-Ozone Overview
-==============
-
-
Ozone is an Object store for Apache Hadoop. It aims to scale to billions of
-keys.
The following is a high-level overview of the core components of Ozone.
-
-
-
-The main elements of Ozone are
:
-
-### Clients
-Ozone ships with a set of ready-made clients. They are
Ozone CLI and Freon.
-
- * [Ozone CLI](./OzoneCommandShell.html) is the command line interface like 'hdfs' command.
-
- * Freon is a load generation tool for Ozone.
-
-### REST Handler
-Ozone provides both an RPC (Remote Procedure Call) as well as a REST
-(Representational State Transfer) style interface. This allows clients to be
-written in many languages quickly. Ozone strives to maintain a similar
-interface between REST and RPC. The Rest handler offers the REST protocol
-services of Ozone.
-
-For most purposes, a client can make one line change to switch from REST to
-RPC or vice versa.
-
-### Ozone File System
-Ozone file system (TODO: Add documentation) is a Hadoop compatible file system.
-This is the important user-visible component of ozone.
-This allows Hadoop services and applications like Hive/Spark to run against
-Ozone without any change.
-
-### Ozone Client
-This is like DFSClient in HDFS. This acts as the standard client to talk to
-Ozone. All other components that we have discussed so far rely on Ozone client
-(TODO: Add Ozone client documentation).
-
-### Key Space Manager
-Key Space Manager(KSM) takes care of the Ozone's namespace.
-All ozone entities like volumes, buckets and keys are managed by KSM
-(TODO: Add KSM documentation). In Short, KSM is the metadata manager for Ozone.
-KSM talks to blockManager(SCM) to get blocks and passes it on to the Ozone
-client. Ozone client writes data to these blocks.
-KSM will eventually be replicated via Apache Ratis for High Availability.
-
-### Storage Container Manager
-Storage Container Manager (SCM) is the block and cluster manager for Ozone.
-SCM along with data nodes offer a service called 'containers'.
-A container is a group unrelated of blocks that are managed together
-as a single entity.
-
-SCM offers the following abstractions.
-
-
-#### Blocks
-Blocks are like blocks in HDFS. They are replicated store of data.
-
-#### Containers
-A collection of blocks replicated and managed together.
-
-#### Pipelines
-SCM allows each container to choose its method of replication.
-For example, a container might decide that it needs only one copy of a block
-and might choose a stand-alone pipeline. Another container might want to have
-a very high level of reliability and pick a RATIS based pipeline. In other
-words, SCM allows different kinds of replication strategies to co-exist.
-
-#### Pools
-A group of data nodes is called a pool. For scaling purposes,
-we define a pool as a set of machines. This makes management of datanodes
-easier.
-
-#### Nodes
-The data node where data is stored.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
deleted file mode 100644
index 0294a53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/OzoneRest.md
+++ /dev/null
@@ -1,549 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-Ozone REST API's.
-===================
-
-<!-- MACRO{toc|fromDepth=0|toDepth=1} -->
-
-Overview
---------
-
-The Ozone REST API's allows user to access ozone via REST protocol.
-
-Authentication and Authorization
---------------------
-
-For time being, The default authentication mode of REST API is insecure access
-mode, which is *Simple* mode. Under this mode, ozone server trusts the user
-name specified by client and it does not perform any authentication.
-
-User name can be specified in HTTP header by
-
-* `x-ozone-user: {USER_NAME}`
-
-for example if add following header *x-ozone-user: bilbo* in the HTTP request,
-then operation will be executed as *bilbo* user.
-In *Simple* mode, there is no real authorization either. Client can be
-authorized to obtain administrator privilege by using HTTP header
-
-* `Authorization: {AUTH_METHOD} {SIGNATURE}`
-
-for example set following header *Authorization: OZONE root* in the HTTP request,
-then ozone will authorize the client with administrator privilege.
-
-Common REST Headers
---------------------
-
-The following HTTP headers must be set for each REST call.
-
-| Property | Description |
-|:---- |:----
-| Authorization | The authorization field determines which authentication method is used by ozone. Currently only *simple* mode is supported, the corresponding value is *OZONE*. Optionally an user name can be set as *OZONE {USER_NAME}* to authorize as a particular user. |
-| Date | Standard HTTP header that represents dates. The format is - day of the week, month, day, year and time (military time format) in GMT. Any other time zone will be rejected by ozone server. Eg. *Date : Mon, Apr 4, 2016 06:22:00 GMT*. This field is required. |
-| x-ozone-version | A required HTTP header to indicate which version of API this call will be communicating to. E.g *x-ozone-version: v1*. Currently ozone only publishes v1 version API. |
-
-Common Reply Headers
---------------------
-
-The common reply headers are part of all Ozone server replies.
-
-| Property | Description |
-|:---- |:----
-| Date | This is the HTTP date header and it is set to server’s local time expressed in GMT. |
-| x-ozone-request-id | This is a UUID string that represents an unique request ID. This ID is used to track the request through the ozone system and is useful for debugging purposes. |
-| x-ozone-server-name | Fully qualified domain name of the sever which handled the request. |
-
-Volume APIs
---------------------
-
-### Create a Volume
-
-This API allows admins to create a new storage volume.
-
-Schema:
-
-- `POST /{volume}?quota=<VOLUME_QUOTA>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| quota | long<BYTES \| MB \| GB \| TB> | Optional. Quota size in BYTEs, MBs, GBs or TBs |
-
-Sample HTTP POST request:
-
- curl -i -X POST -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE root" "http://localhost:9864/volume-to-create"
-
-this request creates a volume as user *bilbo*, the authorization field is set to *OZONE root* because this call requires administration privilege. The client receives a response with zero content length.
-
- HTTP/1.1 201 Created
- x-ozone-server-name: localhost
- x-ozone-request-id: 2173deb5-bbb7-4f0a-8236-f354784e3bae
- Date: Tue, 27 Jun 2017 07:42:04 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-### Update Volume
-
-This API allows administrators to update volume info such as ownership and quota. This API requires administration privilege.
-
-Schema:
-
-- `PUT /{volume}?quota=<VOLUME_QUOTA>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| quota | long<BYTES \| MB \| GB \| TB> \| remove | Optional. Quota size in BYTEs, MBs, GBs or TBs. Or use string value *remove* to remove an existing quota for a volume. |
-
-Sample HTTP PUT request:
-
- curl -X PUT -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user: john" http://localhost:9864/volume-to-update
-
-this request modifies the owner of */volume-to-update* to *john*.
-
-### Delete Volume
-
-This API allows user to delete a volume owned by themselves if the volume is not empty. Administrators can delete volumes owned by any user.
-
-Schema:
-
-- `DELETE /{volume}`
-
-Sample HTTP DELETE request:
-
- curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user: bilbo" http://localhost:9864/volume-to-delete
-
-this request deletes an empty volume */volume-to-delete*. The client receives a zero length content.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: 6af14c64-e3a9-40fe-9634-df60b7cbbc6a
- Date: Tue, 27 Jun 2017 08:49:52 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-### Info Volume
-
-This API allows user to read the info of a volume owned by themselves. Administrators can read volume info owned by any user.
-
-Schema:
-
-- `GET /{volume}?info=volume`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | "volume" | Required and enforced with this value. |
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo?info=volume"
-
-this request gets the info of volume */volume-of-bilbo*, the client receives a response with a JSON object of volume info.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: a2224806-beaf-42dd-a68e-533cd7508f74
- Date: Tue, 27 Jun 2017 07:55:35 GMT
- Content-Type: application/octet-stream
- Content-Length: 171
- Connection: keep-alive
-
- {
- "owner" : { "name" : "bilbo" },
- "quota" : { "unit" : "TB", "size" : 1048576 },
- "volumeName" : "volume-of-bilbo",
- "createdOn" : "Tue, 27 Jun 2017 07:42:04 GMT",
- "createdBy" : "root"
- }
-
-### List Volumes
-
-This API allows user to list all volumes owned by themselves. Administrators can list all volumes owned by any user.
-
-Schema:
-
-- `GET /?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_VOLUME_KEY>`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only volumes with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of volumes included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Volume name from where listing should start, this key is excluded in the result. It must be a valid volume name. |
-| root-scan | bool | Optional. List all volumes in the cluster if this is set to true. Default false. |
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/?max-keys=100&prefix=Jan"
-
-this request gets all volumes owned by *bilbo* and each volume's name contains prefix *Jan*, the result at most contains *100* entries. The client receives a list of SON objects, each of them describes the info of a volume.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: 7fa0dce1-a8bd-4387-bc3c-1dac4b710bb1
- Date: Tue, 27 Jun 2017 08:07:04 GMT
- Content-Type: application/octet-stream
- Content-Length: 602
- Connection: keep-alive
-
- {
- "volumes" : [
- {
- "owner" : { "name" : "bilbo"},
- "quota" : { "unit" : "TB", "size" : 2 },
- "volumeName" : "Jan-vol1",
- "createdOn" : "Tue, 27 Jun 2017 07:42:04 GMT",
- "createdBy" : root
- },
- ...
- ]
- }
-
-Bucket APIs
---------------------
-
-### Create Bucket
-
-This API allows an user to create a bucket in a volume.
-
-Schema:
-
-- `POST /{volume}/{bucket}`
-
-Additional HTTP Headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| x-ozone-acl | ozone ACLs | Optional. Ozone acls. |
-| x-ozone-storage-class | <DEFAULT \| ARCHIVE \| DISK \| RAM_DISK \| SSD > | Optional. Storage type for a volume. |
-| x-ozone-bucket-versioning | enabled/disabled | Optional. Do enable bucket versioning or not. |
-
-Sample HTTP POST request:
-
- curl -i -X POST -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" http://localhost:9864/volume-of-bilbo/bucket-0
-
-this request creates a bucket *bucket-0* under volume *volume-of-bilbo*.
-
- HTTP/1.1 201 Created
- x-ozone-server-name: localhost
- x-ozone-request-id: 49acfeec-4c85-470a-872b-2eaebd8d751e
- Date: Tue, 27 Jun 2017 08:55:25 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-### Update Bucket
-
-Updates bucket meta-data, like ACLs.
-
-Schema:
-
-- `PUT /{volume}/{bucket}`
-
-Additional HTTP Headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| x-ozone-acl | ozone ACLs | Optional. Ozone acls. |
-| x-ozone-bucket-versioning | enabled/disabled | Optional. Do enable bucket versioning or not. |
-
-Sample HTTP PUT request:
-
- curl -i -X PUT -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" -H "x-ozone-acl: ADD user:peregrin:rw" http://localhost:9864/volume-of-bilbo/bucket-to-update
-
-this request adds an ACL policy specified by HTTP header *x-ozone-acl* to bucket */volume-of-bilbo/bucket-to-update*, the ACL field *ADD user:peregrin:rw* gives add additional read/write permission to user *peregrin* to this bucket.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: b061a295-5faf-4b98-94b9-8b3e87c8eb5e
- Date: Tue, 27 Jun 2017 09:02:37 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-### Delete Bucket
-
-Deletes a bucket if it is empty. An user can only delete bucket owned by themselves, and administrators can delete buckets owned by any user, as long as it is empty.
-
-Schema:
-
-- `DELETE /{volume}/{bucket}`
-
-Sample HTTP DELETE request:
-
- curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0"
-
-this request deletes bucket */volume-of-bilbo/bucket-0*. The client receives a zero length content response.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: f57acd7a-2116-4c2f-aa2f-5a483db81c9c
- Date: Tue, 27 Jun 2017 09:16:52 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-
-### Info Bucket
-
-This API returns information about a given bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}?info=bucket`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | "bucket" | Required and enforced with this value. |
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0?info=bucket"
-
-this request gets the info of bucket */volume-of-bilbo/bucket-0*. The client receives a response of JSON object contains bucket info.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: f125485b-8cae-4c7f-a2d6-5b1fefd6f193
- Date: Tue, 27 Jun 2017 09:08:31 GMT
- Content-Type: application/json
- Content-Length: 138
- Connection: keep-alive
-
- {
- "volumeName" : "volume-of-bilbo",
- "bucketName" : "bucket-0",
- "createdOn" : "Tue, 27 Jun 2017 08:55:25 GMT",
- "acls" : [ ],
- "versioning" : "DISABLED",
- "storageType" : "DISK"
- }
-
-### List Buckets
-
-List buckets in a given volume.
-
-Schema:
-
-- `GET /{volume}?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_BUCKET_KEY>`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only buckets with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of buckets included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Bucket name from where listing should start, this key is excluded in the result. It must be a valid bucket name. |
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo?max-keys=10"
-
-this request lists all the buckets under volume *volume-of-bilbo*, and the result at most contains 10 entries. The client receives response of a array of JSON objects, each of them represents for a bucket info.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: e048c3d5-169c-470f-9903-632d9f9e32d5
- Date: Tue, 27 Jun 2017 09:12:18 GMT
- Content-Type: application/octet-stream
- Content-Length: 207
- Connection: keep-alive
-
- {
- "buckets" : [ {
- "volumeName" : "volume-of-bilbo",
- "bucketName" : "bucket-0",
- "createdOn" : "Tue, 27 Jun 2017 08:55:25 GMT",
- "acls" : [ ],
- "versioning" : null,
- "storageType" : "DISK",
- "bytesUsed" : 0,
- "keyCount" : 0
- },
- ...
- ]
- }
-
-Key APIs
-------------------
-
-### Put Key
-
-This API allows user to create or overwrite keys inside of a bucket.
-
-Schema:
-
-- `PUT /{volume}/{bucket}/{key}`
-
-Additional HTTP headers:
-
-| HTTP Header | Value | Description |
-|:---- |:---- |:----
-| Content-MD5 | MD5 digest | Standard HTTP header, file hash. |
-
-Sample PUT HTTP request:
-
- curl -X PUT -T /path/to/localfile -H "Authorization:OZONE" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request uploads a local file */path/to/localfile* specified by option *-T* to ozone as user *bilbo*, mapped to ozone key */volume-of-bilbo/bucket-0/file-0*. The client receives a zero length content response.
-
-### Get Key
-
-This API allows user to get or download a key from an ozone bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}/{key}`
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request reads the content of key */volume-of-bilbo/bucket-0/file-0*. If the content of the file is plain text, it can be directly dumped onto stdout.
-
- HTTP/1.1 200 OK
- Content-Type: application/octet-stream
- x-ozone-server-name: localhost
- x-ozone-request-id: 1bcd7de7-d8e3-46bb-afee-bdc933d383b8
- Date: Tue, 27 Jun 2017 09:35:29 GMT
- Content-Length: 6
- Connection: keep-alive
-
- Hello Ozone!
-
-if the file is not plain text, specify *-O* option in curl command and the file *file-0* will be downloaded into current working directory, file name will be same as the key. A sample request like following:
-
- curl -O -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/bucket-0/file-1"
-
-response looks like following:
-
- % Total % Received % Xferd Average Speed Time Time Time Current
- Dload Upload Total Spent Left Speed
- 100 6148k 100 6148k 0 0 24.0M 0 --:--:-- --:--:-- --:--:-- 24.1M
-
-### Delete Key
-
-This API allows user to delete a key from a bucket.
-
-Schema:
-
-- `DELETE /{volume}/{bucket}/{key}`
-
-Sample HTTP DELETE request:
-
- curl -i -X DELETE -H "Authorization:OZONE root" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "x-ozone-version: v1" -H "x-ozone-user:bilbo" "http://localhost:9864/volume-of-bilbo/bucket-0/file-0"
-
-this request deletes key */volume-of-bilbo/bucket-0/file-0*. The client receives a zero length content result:
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: f8c4a373-dd5f-4e3a-b6c4-ddf7e191fe91
- Date: Tue, 27 Jun 2017 14:19:48 GMT
- Content-Type: application/octet-stream
- Content-Length: 0
- Connection: keep-alive
-
-### Info Key
-
-This API returns information about a given key.
-
-Schema:
-
-- `GET /{volume}/{bucket}/{key}?info=key`
-
-Query Parameter:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| info | String, "key" | Required and enforced with this value. |
-
-Sample HTTP DELETE request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http://localhost:9864/volume-of-bilbo/buket-0/file-0?info=key"
-
-this request returns information of the key */volume-of-bilbo/bucket-0/file-0*. The client receives a JSON object listed attributes of the key.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: c674343c-a0f2-49e4-bbd6-daa73e7dc131
- Date: Mon, 03 Jul 2017 14:28:45 GMT
- Content-Type: application/octet-stream
- Content-Length: 73
- Connection: keep-alive
-
- {
- "version" : 0,
- "md5hash" : null,
- "createdOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
- "modifiedOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
- "size" : 0,
- "keyName" : "file-0"
- }
-
-### List Keys
-
-This API allows user to list keys in a bucket.
-
-Schema:
-
-- `GET /{volume}/{bucket}?prefix=<PREFIX>&max-keys=<MAX_RESULT_SIZE>&prev-key=<PREVIOUS_KEY>`
-
-Query Parameters:
-
-| Query Parameter | Value | Description |
-|:---- |:---- |:----
-| prefix | string | Optional. Only keys with this prefix are included in the result. |
-| max-keys | int | Optional. Maximum number of keys included in the result. Default is 1024 if not specified. |
-| prev-key | string | Optional. Key name from where listing should start, this key is excluded in the result. It must be a valid key name. |
-
-Sample HTTP GET request:
-
- curl -i -H "x-ozone-user: bilbo" -H "x-ozone-version: v1" -H "Date: Mon, 26 Jun 2017 04:23:30 GMT" -H "Authorization:OZONE" "http:/localhost:9864/volume-of-bilbo/bucket-0/?max-keys=100&prefix=file"
-
-this request list keys under bucket */volume-of-bilbo/bucket-0*, the listing result is filtered by prefix *file*. The client receives an array of JSON objects, each of them represents the info of a matched key.
-
- HTTP/1.1 200 OK
- x-ozone-server-name: localhost
- x-ozone-request-id: 7f9fc970-9904-4c56-b671-83a086c6f555
- Date: Tue, 27 Jun 2017 09:48:59 GMT
- Content-Type: application/json
- Content-Length: 209
- Connection: keep-alive
-
- {
- "name" : null,
- "prefix" : file,
- "maxKeys" : 0,
- "truncated" : false,
- "keyList" : [ {
- "version" : 0,
- "md5hash" : null,
- "createdOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
- "modifiedOn" : "Mon, 26 Jun 2017 04:23:30 GMT",
- "size" : 0,
- "keyName" : "file-0"
- },
- ...
- ]
- }
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
deleted file mode 100644
index 158b5bb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/Shell.java
+++ /dev/null
@@ -1,415 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.bucket.UpdateBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.DeleteKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.GetKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.InfoKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.ListKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.keys.PutKeyHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.CreateVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.DeleteVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.InfoVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.ListVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.volume.UpdateVolumeHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.CreateBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.DeleteBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.InfoBucketHandler;
-import org.apache.hadoop.ozone.web.ozShell.bucket.ListBucketHandler;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.URISyntaxException;
-
-/**
- * Ozone user interface commands.
- *
- * This class uses dispatch method to make calls
- * to appropriate handlers that execute the ozone functions.
- */
-public class Shell extends Configured implements Tool {
- private static final Logger LOG = LoggerFactory.getLogger(Shell.class);
-
- // General options
- public static final int DEFAULT_OZONE_PORT = 50070;
- public static final String VERBOSE = "v";
-
- // volume related command line arguments
- public static final String RUNAS = "root";
- public static final String USER = "user";
- public static final String OWNER = "owner";
- public static final String QUOTA = "quota";
- public static final String CREATE_VOLUME = "createVolume";
- public static final String UPDATE_VOLUME = "updateVolume";
- public static final String DELETE_VOLUME = "deleteVolume";
- public static final String LIST_VOLUME = "listVolume";
- public static final String INFO_VOLUME = "infoVolume";
-
- // bucket related command line arguments
- public static final String CREATE_BUCKET = "createBucket";
- public static final String UPDATE_BUCKET = "updateBucket";
- public static final String DELETE_BUCKET = "deleteBucket";
- public static final String LIST_BUCKET = "listBucket";
- public static final String INFO_BUCKET = "infoBucket";
- public static final String ADD_ACLS = "addAcl";
- public static final String REMOVE_ACLS = "removeAcl";
- // TODO : Support versioning and StorageType for buckets
-
- //Object related command line arguments
- public static final String PUT_KEY = "putKey";
- public static final String GET_KEY = "getKey";
- public static final String INFO_KEY = "infoKey";
- public static final String DELETE_KEY = "deleteKey";
- public static final String LIST_KEY = "listKey";
- public static final String FILE = "file";
-
- // Listing related command line arguments
- public static final String LIST_LENGTH = "length";
- public static final String START = "start";
- public static final String PREFIX = "prefix";
-
- /**
- * Main for the ozShell Command handling.
- *
- * @param argv - System Args Strings[]
- * @throws Exception
- */
- public static void main(String[] argv) throws Exception {
- Shell shell = new Shell();
- Configuration conf = new OzoneConfiguration();
- conf.setQuietMode(false);
- shell.setConf(conf);
- int res = 0;
- try {
- res = ToolRunner.run(shell, argv);
- } catch (Exception ex) {
- System.err.println("ERROR: " + ex.getMessage());
- System.exit(1);
- }
- System.exit(res);
- }
-
- /**
- * Execute the command with the given arguments.
- *
- * @param args command specific arguments.
- *
- * @return exit code.
- *
- * @throws Exception
- */
- @Override
- public int run(String[] args) throws Exception {
- Options opts = getOpts();
- CommandLine cmd = parseArgs(args, opts);
- return dispatch(cmd, opts);
- }
-
- /**
- * returns the Command Line Options.
- *
- * @return Options
- */
- private Options getOpts() {
- Options opts = new Options();
- addVolumeCommands(opts);
- addBucketCommands(opts);
- addKeyCommands(opts);
- addListingCommands(opts);
- return opts;
- }
-
- /**
- * This function parses all command line arguments
- * and returns the appropriate values.
- *
- * @param argv - Argv from main
- *
- * @return CommandLine
- */
- private CommandLine parseArgs(String[] argv, Options opts)
- throws org.apache.commons.cli.ParseException {
- try {
- BasicParser parser = new BasicParser();
- return parser.parse(opts, argv);
- } catch (ParseException ex) {
- System.out.printf(ex.getMessage());
- }
-
- return null;
- }
-
-
- /**
- * All volume related commands are added in this function for the command
- * parser.
- *
- * @param options - Command Options class.
- */
- private void addVolumeCommands(Options options) {
- Option verbose = new Option(VERBOSE, false, "verbose information output.");
- options.addOption(verbose);
-
- Option runas = new Option(RUNAS, false, "Run the command as \"hdfs\" user");
- options.addOption(runas);
-
- Option userName = new Option(USER, true,
- "Name of the user in volume management " +
- "functions");
- options.addOption(userName);
-
- Option quota = new Option(QUOTA, true, "Quota for the volume. E.g. 10TB");
- options.addOption(quota);
-
-
- Option createVolume = new Option(CREATE_VOLUME, true, "creates a volume" +
- "for the specified user.\n \t For example : hdfs oz -createVolume " +
- "<volumeURI> -root -user <userName>\n");
- options.addOption(createVolume);
-
- Option deleteVolume = new Option(DELETE_VOLUME, true, "deletes a volume" +
- "if it is empty.\n \t For example : hdfs oz -deleteVolume <volumeURI>" +
- " -root \n");
- options.addOption(deleteVolume);
-
- Option listVolume =
- new Option(LIST_VOLUME, true, "List the volumes of a given user.\n" +
- "For example : hdfs oz -listVolume <ozoneURI>" +
- "-user <username> -root or hdfs oz " +
- "-listVolume");
- options.addOption(listVolume);
-
- Option updateVolume =
- new Option(UPDATE_VOLUME, true, "updates an existing volume.\n" +
- "\t For example : hdfs oz " +
- "-updateVolume <volumeURI> -quota " +
- "100TB\n");
- options.addOption(updateVolume);
-
- Option infoVolume = new Option(INFO_VOLUME, true,
- "returns information about a specific " +
- "volume.");
- options.addOption(infoVolume);
- }
-
- /**
- * All bucket related commands for ozone.
- *
- * @param opts - Options
- */
- private void addBucketCommands(Options opts) {
- Option createBucket = new Option(CREATE_BUCKET, true,
- "creates a bucket in a given volume." +
- "For example: hdfs oz -createBucket <bucketURI>");
- opts.addOption(createBucket);
-
- Option infoBucket =
- new Option(INFO_BUCKET, true, "returns information about a bucket.");
- opts.addOption(infoBucket);
-
- Option deleteBucket =
- new Option(DELETE_BUCKET, true, "deletes an empty bucket.");
- opts.addOption(deleteBucket);
-
- Option listBucket =
- new Option(LIST_BUCKET, true, "lists the buckets in a volume.");
- opts.addOption(listBucket);
-
- Option updateBucket =
- new Option(UPDATE_BUCKET, true, "allows changing bucket attributes.\n" +
- " For example: hdfs oz -updateBucket <bucketURI> " +
- "-addAcl user:frodo:rw");
- opts.addOption(updateBucket);
-
- Option addAcl =
- new Option(ADD_ACLS, true, "allows user to add acls to a bucket.");
- opts.addOption(addAcl);
-
- Option removeAcl =
- new Option(REMOVE_ACLS, true, "allows user to remove acls from a " +
- "bucket.");
- opts.addOption(removeAcl);
- }
-
- /**
- * All key commands.
- *
- * @param opts - options
- */
- private void addKeyCommands(Options opts) {
- Option putKey =
- new Option(PUT_KEY, true, "creates or overwrites an existing key");
- opts.addOption(putKey);
-
- Option deleteKey =
- new Option(DELETE_KEY, true, "deletes an existing key");
- opts.addOption(deleteKey);
-
- Option infoKey =
- new Option(INFO_KEY, true, "returns information about an existing key");
- opts.addOption(infoKey);
-
- Option listKey =
- new Option(LIST_KEY, true, "list all keys in a given bucket");
- opts.addOption(listKey);
-
- Option getKey =
- new Option(GET_KEY, true, "Gets a specific key from ozone server.");
- opts.addOption(getKey);
-
- Option fileArgument =
- new Option(FILE, true, "Data file path");
- opts.addOption(fileArgument);
-
- }
-
- /**
- * Sub commands for list command.
- * @param opts
- */
- private void addListingCommands(Options opts) {
- Option maxKeys = new Option(LIST_LENGTH, true,
- "Specify the max length of listing result.");
- opts.addOption(maxKeys);
-
- Option prevKey = new Option(START, true,
- "Specify the start key where to start listing from.");
- opts.addOption(prevKey);
-
- Option prefix = new Option(PREFIX, true,
- "Specify the prefix to filter the listing result.");
- opts.addOption(prefix);
- }
-
- /**
- * Dispatches calls to the right command Handler classes.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- private int dispatch(CommandLine cmd, Options opts)
- throws IOException, OzoneException, URISyntaxException {
- Handler handler = null;
- final int eightyColumn = 80;
-
- try {
-
- // volume functions
- if (cmd.hasOption(Shell.CREATE_VOLUME)) {
- handler = new CreateVolumeHandler();
- }
-
- if (cmd.hasOption(Shell.DELETE_VOLUME)) {
- handler = new DeleteVolumeHandler();
- }
-
- if (cmd.hasOption(Shell.LIST_VOLUME)) {
- handler = new ListVolumeHandler();
- }
-
- if (cmd.hasOption(Shell.UPDATE_VOLUME)) {
- handler = new UpdateVolumeHandler();
- }
-
- if (cmd.hasOption(Shell.INFO_VOLUME)) {
- handler = new InfoVolumeHandler();
- }
-
- // bucket functions
- if (cmd.hasOption(Shell.CREATE_BUCKET)) {
- handler = new CreateBucketHandler();
- }
-
- if (cmd.hasOption(Shell.DELETE_BUCKET)) {
- handler = new DeleteBucketHandler();
- }
-
- if (cmd.hasOption(Shell.INFO_BUCKET)) {
- handler = new InfoBucketHandler();
- }
-
- if (cmd.hasOption(Shell.LIST_BUCKET)) {
- handler = new ListBucketHandler();
- }
-
- if(cmd.hasOption(Shell.UPDATE_BUCKET)){
- handler = new UpdateBucketHandler();
- }
-
- //Key Functions
-
- if(cmd.hasOption(Shell.PUT_KEY)) {
- handler = new PutKeyHandler();
- }
-
- if(cmd.hasOption(Shell.DELETE_KEY)) {
- handler = new DeleteKeyHandler();
- }
-
- if(cmd.hasOption(Shell.INFO_KEY)) {
- handler = new InfoKeyHandler();
- }
-
- if(cmd.hasOption(Shell.LIST_KEY)) {
- handler = new ListKeyHandler();
- }
-
- if(cmd.hasOption(Shell.GET_KEY)) {
- handler = new GetKeyHandler();
- }
-
- if (handler != null) {
- handler.execute(cmd);
- return 0;
- } else {
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.printHelp(eightyColumn, "hdfs oz -command uri [args]",
- "Ozone Commands",
- opts, "Please correct your command and try again.");
- return 1;
- }
- } catch (IOException | URISyntaxException ex) {
- System.err.printf("Command Failed : %s%n", ex.getMessage());
- } catch (OzoneException ex) {
- System.err.printf("Command Failed : %s%n", ex.toJsonString());
- LOG.debug("Command Failed.", ex);
- } catch (IllegalArgumentException ex) {
- System.err.printf("Illegal argument: %s%n", ex.getMessage());
- }
- return 1;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
deleted file mode 100644
index d1c46b5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/CreateBucketHandler.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * create bucket handler.
- */
-public class CreateBucketHandler extends Handler {
-
- private String volumeName;
- private String bucketName;
- private String rootName;
-
- /**
- * Executes create bucket.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.CREATE_BUCKET)) {
- throw new OzoneRestClientException(
- "Incorrect call : createBucket is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.CREATE_BUCKET);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
- "volume and bucket name required in createBucket");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
-
- OzoneVolume vol = client.getVolume(volumeName);
- OzoneBucket bucket = vol.createBucket(bucketName);
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- bucket.getBucketInfo().toJsonString()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
deleted file mode 100644
index 181f6cc..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/DeleteBucketHandler.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Delete bucket Handler.
- */
-public class DeleteBucketHandler extends Handler {
- private String volumeName;
- private String bucketName;
- private String rootName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.DELETE_BUCKET)) {
- throw new OzoneRestClientException(
- "Incorrect call : deleteBucket is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.DELETE_BUCKET);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
- "volume and bucket name required in delete Bucket");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- OzoneVolume vol = client.getVolume(volumeName);
- vol.deleteBucket(bucketName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
deleted file mode 100644
index 321c4c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/InfoBucketHandler.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Executes Info bucket.
- */
-public class InfoBucketHandler extends Handler {
- private String volumeName;
- private String bucketName;
- private String rootName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.INFO_BUCKET)) {
- throw new OzoneRestClientException(
- "Incorrect call : infoBucket is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.INFO_BUCKET);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
-
- if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
- "volume and bucket name required in info Bucket");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- OzoneVolume vol = client.getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
-
- System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- bucket.getBucketInfo().toJsonString()));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
deleted file mode 100644
index ea77cae..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/ListBucketHandler.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Bucket.
- */
-public class ListBucketHandler extends Handler {
- private String volumeName;
- private String rootName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.LIST_BUCKET)) {
- throw new OzoneRestClientException(
- "Incorrect call : listBucket is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.LIST_BUCKET);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 1) {
- throw new OzoneRestClientException("volume is required in listBucket");
- }
-
- volumeName = path.getName(0).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- String length = null;
- if (cmd.hasOption(Shell.LIST_LENGTH)) {
- length = cmd.getOptionValue(Shell.LIST_LENGTH);
- OzoneUtils.verifyMaxKeyLength(length);
- }
-
- String startBucket = null;
- if (cmd.hasOption(Shell.START)) {
- startBucket = cmd.getOptionValue(Shell.START);
- }
-
- String prefix = null;
- if (cmd.hasOption(Shell.PREFIX)) {
- prefix = cmd.getOptionValue(Shell.PREFIX);
- }
-
- OzoneVolume vol = client.getVolume(volumeName);
- List<OzoneBucket> bucketList = vol.listBuckets(length, startBucket, prefix);
-
- List<BucketInfo> jsonData = bucketList.stream()
- .map(OzoneBucket::getBucketInfo).collect(Collectors.toList());
- System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- JsonUtils.toJsonString(jsonData)));
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
deleted file mode 100644
index 781a00c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/UpdateBucketHandler.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Allows users to add and remove acls and from a bucket.
- */
-public class UpdateBucketHandler extends Handler {
- private String volumeName;
- private String bucketName;
- private String rootName;
-
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.UPDATE_BUCKET)) {
- throw new OzoneRestClientException(
- "Incorrect call : updateBucket is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.UPDATE_BUCKET);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
-
- if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
- "volume and bucket name required in update bucket");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- OzoneVolume vol = client.getVolume(volumeName);
- if (cmd.hasOption(Shell.ADD_ACLS)) {
- String aclString = cmd.getOptionValue(Shell.ADD_ACLS);
- String[] aclArray = aclString.split(",");
- vol.addAcls(bucketName, aclArray);
- }
-
- if (cmd.hasOption(Shell.REMOVE_ACLS)) {
- String aclString = cmd.getOptionValue(Shell.REMOVE_ACLS);
- String[] aclArray = aclString.split(",");
- vol.removeAcls(bucketName, aclArray);
- }
-
- OzoneBucket bucket = vol.getBucket(bucketName);
- System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- bucket.getBucketInfo().toJsonString()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
deleted file mode 100644
index c344c35..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/bucket/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.bucket;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
deleted file mode 100644
index 1ad2588..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/DeleteKeyHandler.java
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Executes Delete Key.
- */
-public class DeleteKeyHandler extends Handler {
- private String userName;
- private String volumeName;
- private String bucketName;
- private String keyName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.DELETE_KEY)) {
- throw new OzoneRestClientException(
- "Incorrect call : deleteKey is missing");
- }
-
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
-
- String ozoneURIString = cmd.getOptionValue(Shell.DELETE_KEY);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 3) {
- throw new OzoneRestClientException(
- "volume/bucket/key name required in deleteKey");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
- keyName = path.getName(2).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(userName);
-
-
- OzoneVolume vol = client.getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- bucket.deleteKey(keyName);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
deleted file mode 100644
index a56bbc0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/GetKeyHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Gets an existing key.
- */
-public class GetKeyHandler extends Handler {
- private String userName;
- private String volumeName;
- private String bucketName;
- private String keyName;
-
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.GET_KEY)) {
- throw new OzoneRestClientException("Incorrect call : getKey is missing");
- }
-
- if (!cmd.hasOption(Shell.FILE)) {
- throw new OzoneRestClientException(
- "get key needs a file path to download to");
- }
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
-
- String ozoneURIString = cmd.getOptionValue(Shell.GET_KEY);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 3) {
- throw new OzoneRestClientException(
- "volume/bucket/key name required in putKey");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
- keyName = path.getName(2).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
-
- String fileName = cmd.getOptionValue(Shell.FILE);
- Path dataFilePath = Paths.get(fileName);
- File dataFile = new File(fileName);
-
-
- if (dataFile.exists()) {
- throw new OzoneRestClientException(fileName +
- "exists. Download will overwrite an " +
- "existing file. Aborting.");
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(userName);
-
- client.getKey(volumeName, bucketName, keyName, dataFilePath);
- if(cmd.hasOption(Shell.VERBOSE)) {
- FileInputStream stream = new FileInputStream(dataFile);
- String hash = DigestUtils.md5Hex(stream);
- System.out.printf("Downloaded file hash : %s%n", hash);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
deleted file mode 100644
index 3e9b2da..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/InfoKeyHandler.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneBucket;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-/**
- * Executes Info Object.
- */
-public class InfoKeyHandler extends Handler {
- private String userName;
- private String volumeName;
- private String bucketName;
- private String keyName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.INFO_KEY)) {
- throw new OzoneRestClientException("Incorrect call : infoKey is missing");
- }
-
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
-
- String ozoneURIString = cmd.getOptionValue(Shell.INFO_KEY);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 3) {
- throw new OzoneRestClientException(
- "volume/bucket/key name required in infoKey");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
- keyName = path.getName(2).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(userName);
-
- OzoneVolume vol = client.getVolume(volumeName);
- OzoneBucket bucket = vol.getBucket(bucketName);
- OzoneKey key = bucket.getKeyInfo(keyName);
-
- System.out.printf("%s%n", JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- key.getObjectInfo().toJsonString()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
deleted file mode 100644
index ea563ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/ListKeyHandler.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneKey;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Keys.
- */
-public class ListKeyHandler extends Handler {
- private String userName;
- private String volumeName;
- private String bucketName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
-
- if (!cmd.hasOption(Shell.LIST_KEY)) {
- throw new OzoneRestClientException(
- "Incorrect call : listKey is missing");
- }
-
- String length = null;
- if (cmd.hasOption(Shell.LIST_LENGTH)) {
- length = cmd.getOptionValue(Shell.LIST_LENGTH);
- OzoneUtils.verifyMaxKeyLength(length);
- }
-
- String startKey = null;
- if (cmd.hasOption(Shell.START)) {
- startKey = cmd.getOptionValue(Shell.START);
- }
-
- String prefix = null;
- if (cmd.hasOption(Shell.PREFIX)) {
- prefix = cmd.getOptionValue(Shell.PREFIX);
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.LIST_KEY);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 2) {
- throw new OzoneRestClientException(
- "volume/bucket is required in listKey");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("bucket Name : %s%n", bucketName);
- }
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(userName);
-
- List<OzoneKey> keys = client.listKeys(volumeName, bucketName, length,
- startKey, prefix);
-
- List<KeyInfo> jsonData = keys.stream()
- .map(OzoneKey::getObjectInfo).collect(Collectors.toList());
- System.out.printf(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- JsonUtils.toJsonString(jsonData)));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
deleted file mode 100644
index 1f2c692..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/PutKeyHandler.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.keys;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
-/**
- * Puts a file into an ozone bucket.
- */
-public class PutKeyHandler extends Handler {
- private String userName;
- private String volumeName;
- private String bucketName;
- private String keyName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.PUT_KEY)) {
- throw new OzoneRestClientException("Incorrect call : putKey is missing");
- }
-
- if (!cmd.hasOption(Shell.FILE)) {
- throw new OzoneRestClientException("put key needs a file to put");
- }
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.PUT_KEY);
- URI ozoneURI = verifyURI(ozoneURIString);
- Path path = Paths.get(ozoneURI.getPath());
- if (path.getNameCount() < 3) {
- throw new OzoneRestClientException(
- "volume/bucket/key name required in putKey");
- }
-
- volumeName = path.getName(0).toString();
- bucketName = path.getName(1).toString();
- keyName = path.getName(2).toString();
-
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume Name : %s%n", volumeName);
- System.out.printf("Bucket Name : %s%n", bucketName);
- System.out.printf("Key Name : %s%n", keyName);
- }
-
-
- String fileName = cmd.getOptionValue(Shell.FILE);
- File dataFile = new File(fileName);
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- FileInputStream stream = new FileInputStream(dataFile);
- String hash = DigestUtils.md5Hex(stream);
- System.out.printf("File Hash : %s%n", hash);
- stream.close();
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(userName);
-
- client.putKey(volumeName, bucketName, keyName, dataFile);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
deleted file mode 100644
index 1deb7ad..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/keys/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.keys;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
deleted file mode 100644
index e33b6e7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * ozShell Class acts as the command line interface to
- * the ozone Rest Client.
- */
-package org.apache.hadoop.ozone.web.ozShell;
-
-/**
- A simple CLI to work against Ozone.
- **/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
deleted file mode 100644
index c0b0bb9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/CreateVolumeHandler.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes the create volume call for the shell.
- */
-public class CreateVolumeHandler extends Handler {
-
- private String rootName;
- private String userName;
- private String volumeName;
- private String quota;
-
- /**
- * Executes the Create Volume.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.CREATE_VOLUME)) {
- throw new OzoneRestClientException(
- "Incorrect call : createVolume is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.CREATE_VOLUME);
- URI ozoneURI = verifyURI(ozoneURIString);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneRestClientException(
- "Volume name is required to create a volume");
- }
-
- // we need to skip the slash in the URI path
- // getPath returns /volumeName needs to remove the first slash.
- volumeName = ozoneURI.getPath().substring(1);
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume name : %s%n", volumeName);
- }
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- if (!cmd.hasOption(Shell.USER)) {
- throw new OzoneRestClientException(
- "User name is needed in createVolume call.");
- }
-
- if (cmd.hasOption(Shell.QUOTA)) {
- quota = cmd.getOptionValue(Shell.QUOTA);
- }
-
- userName = cmd.getOptionValue(Shell.USER);
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- OzoneVolume vol = client.createVolume(volumeName, userName, quota);
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("%s%n",
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(vol.getJsonString()));
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
deleted file mode 100644
index bed7db3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/DeleteVolumeHandler.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes deleteVolume call for the shell.
- */
-public class DeleteVolumeHandler extends Handler {
-
- private String volumeName;
- private String rootName;
-
- /**
- * Executes the delete volume call.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
-
- if (!cmd.hasOption(Shell.DELETE_VOLUME)) {
- throw new OzoneRestClientException(
- "Incorrect call : deleteVolume call is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.DELETE_VOLUME);
- URI ozoneURI = verifyURI(ozoneURIString);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneRestClientException(
- "Volume name is required to delete a volume");
- }
-
- // we need to skip the slash in the URI path
- volumeName = ozoneURI.getPath().substring(1);
-
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Volume name : %s%n", volumeName);
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
- client.deleteVolume(volumeName);
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
deleted file mode 100644
index 16de3d4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/InfoVolumeHandler.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes volume Info calls.
- */
-public class InfoVolumeHandler extends Handler{
-
- private String rootName;
- private String volumeName;
- private String userName;
-
- /**
- * Executes volume Info.
- *
- * @param cmd - CommandLine
- *
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
-
- if (!cmd.hasOption(Shell.INFO_VOLUME)) {
- throw new OzoneRestClientException(
- "Incorrect call : infoVolume is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.INFO_VOLUME);
- URI ozoneURI = verifyURI(ozoneURIString);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneRestClientException(
- "Volume name is required to get info of a volume");
- }
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- }
-
- // we need to skip the slash in the URI path
- volumeName = ozoneURI.getPath().substring(1);
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
-
- if (rootName != null) {
- client.setUserAuth(rootName);
- } else {
- client.setUserAuth(userName);
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- OzoneVolume vol = client.getVolume(volumeName);
- System.out.printf("%s%n",
- JsonUtils.toJsonStringWithDefaultPrettyPrinter(vol.getJsonString()));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
deleted file mode 100644
index 189c891..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/ListVolumeHandler.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.web.client.OzoneVolume;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Executes List Volume call.
- */
-public class ListVolumeHandler extends Handler {
- private String rootName;
- private String userName;
-
- /**
- * Executes the Client Calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
-
- if (!cmd.hasOption(Shell.LIST_VOLUME)) {
- throw new OzoneRestClientException(
- "Incorrect call : listVolume is missing");
- }
-
- int maxKeys = 0;
- if (cmd.hasOption(Shell.LIST_LENGTH)) {
- String length = cmd.getOptionValue(Shell.LIST_LENGTH);
- OzoneUtils.verifyMaxKeyLength(length);
-
- maxKeys = Integer.parseInt(length);
- }
-
- String startVolume = null;
- if (cmd.hasOption(Shell.START)) {
- startVolume = cmd.getOptionValue(Shell.START);
- }
-
- String prefix = null;
- if (cmd.hasOption(Shell.PREFIX)) {
- prefix = cmd.getOptionValue(Shell.PREFIX);
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.LIST_VOLUME);
- URI ozoneURI = verifyURI(ozoneURIString);
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- }
-
- if (cmd.hasOption(Shell.USER)) {
- userName = cmd.getOptionValue(Shell.USER);
- } else {
- userName = System.getProperty("user.name");
- }
-
- client.setEndPointURI(ozoneURI);
- if (rootName != null) {
- client.setUserAuth(rootName);
- } else {
- client.setUserAuth(userName);
- }
-
- List<OzoneVolume> volumes = client.listVolumes(userName, prefix, maxKeys,
- startVolume);
- if (volumes != null) {
- if (cmd.hasOption(Shell.VERBOSE)) {
- System.out.printf("Found : %d volumes for user : %s %n", volumes.size(),
- userName);
- }
-
- List<VolumeInfo> jsonData = volumes.stream()
- .map(OzoneVolume::getVolumeInfo).collect(Collectors.toList());
- System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(
- JsonUtils.toJsonString(jsonData)));
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
deleted file mode 100644
index 164fe34..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/UpdateVolumeHandler.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.ozShell.volume;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.hadoop.ozone.web.client.OzoneRestClientException;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.ozShell.Handler;
-import org.apache.hadoop.ozone.web.ozShell.Shell;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.URISyntaxException;
-
-/**
- * Executes update volume calls.
- */
-public class UpdateVolumeHandler extends Handler {
- private String rootName;
- private String ownerName;
- private String volumeName;
- private String quota;
-
- /**
- * Executes update volume calls.
- *
- * @param cmd - CommandLine
- * @throws IOException
- * @throws OzoneException
- * @throws URISyntaxException
- */
- @Override
- protected void execute(CommandLine cmd)
- throws IOException, OzoneException, URISyntaxException {
- if (!cmd.hasOption(Shell.UPDATE_VOLUME)) {
- throw new OzoneRestClientException(
- "Incorrect call : updateVolume is missing");
- }
-
- String ozoneURIString = cmd.getOptionValue(Shell.UPDATE_VOLUME);
- URI ozoneURI = verifyURI(ozoneURIString);
- if (ozoneURI.getPath().isEmpty()) {
- throw new OzoneRestClientException(
- "Volume name is required to update a volume");
- }
-
- // we need to skip the slash in the URI path
- volumeName = ozoneURI.getPath().substring(1);
-
- if (cmd.hasOption(Shell.RUNAS)) {
- rootName = "hdfs";
- } else {
- rootName = System.getProperty("user.name");
- }
-
- if (cmd.hasOption(Shell.QUOTA)) {
- quota = cmd.getOptionValue(Shell.QUOTA);
- }
-
- if (cmd.hasOption(Shell.USER)) {
- ownerName = cmd.getOptionValue(Shell.USER);
- }
-
- client.setEndPointURI(ozoneURI);
- client.setUserAuth(rootName);
-
- if (quota != null && !quota.isEmpty()) {
- client.setVolumeQuota(volumeName, quota);
- }
-
- if (ownerName != null && !ownerName.isEmpty()) {
- client.setVolumeOwner(volumeName, ownerName);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
deleted file mode 100644
index fc19274..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ozShell/volume/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.ozShell.volume;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
deleted file mode 100644
index 1a7275c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-/**
- * This package contains generic class for the internal http server
- * and REST interfaces.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
deleted file mode 100644
index 9619ebd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/OzoneQuota.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.request;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-
-/**
- * represents an OzoneQuota Object that can be applied to
- * a storage volume.
- */
-@InterfaceAudience.Private
-public class OzoneQuota {
-
- private Units unit;
- private int size;
-
- /** Quota Units.*/
- public enum Units {UNDEFINED, BYTES, MB, GB, TB}
-
- /**
- * Returns size.
- *
- * @return int
- */
- public int getSize() {
- return size;
- }
-
- /**
- * Returns Units.
- *
- * @return Unit in MB, GB or TB
- */
- public Units getUnit() {
- return unit;
- }
-
- /**
- * Constructs a default Quota object.
- */
- public OzoneQuota() {
- this.size = 0;
- this.unit = Units.UNDEFINED;
- }
-
- /**
- * Constructor for Ozone Quota.
- *
- * @param size - Integer Size
- * @param unit MB, GB or TB
- */
- public OzoneQuota(int size, Units unit) {
- this.size = size;
- this.unit = unit;
- }
-
- /**
- * Formats a quota as a string.
- *
- * @param quota the quota to format
- * @return string representation of quota
- */
- public static String formatQuota(OzoneQuota quota) {
- return String.valueOf(quota.size) + quota.unit;
- }
-
- /**
- * Parses a user provided string and returns the
- * Quota Object.
- *
- * @param quotaString Quota String
- *
- * @return OzoneQuota object
- *
- * @throws IllegalArgumentException
- */
- public static OzoneQuota parseQuota(String quotaString)
- throws IllegalArgumentException {
-
- if ((quotaString == null) || (quotaString.isEmpty())) {
- throw new IllegalArgumentException(
- "Quota string cannot be null or empty.");
- }
-
- if (isRemove(quotaString)) {
- throw new IllegalArgumentException("Remove is invalid in this context.");
- }
-
- String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
- String size = "";
- int nSize;
- Units currUnit = Units.MB;
- Boolean found = false;
- if (uppercase.endsWith(Header.OZONE_QUOTA_MB)) {
- size = uppercase
- .substring(0, uppercase.length() - Header.OZONE_QUOTA_MB.length());
- currUnit = Units.MB;
- found = true;
- }
-
- if (uppercase.endsWith(Header.OZONE_QUOTA_GB)) {
- size = uppercase
- .substring(0, uppercase.length() - Header.OZONE_QUOTA_GB.length());
- currUnit = Units.GB;
- found = true;
- }
-
- if (uppercase.endsWith(Header.OZONE_QUOTA_TB)) {
- size = uppercase
- .substring(0, uppercase.length() - Header.OZONE_QUOTA_TB.length());
- currUnit = Units.TB;
- found = true;
- }
-
- if (uppercase.endsWith(Header.OZONE_QUOTA_BYTES)) {
- size = uppercase
- .substring(0, uppercase.length() - Header.OZONE_QUOTA_BYTES.length());
- currUnit = Units.BYTES;
- found = true;
- }
-
- if (!found) {
- throw new IllegalArgumentException(
- "Quota unit not recognized. Supported values are BYTES, MB, GB and " +
- "TB.");
- }
-
- nSize = Integer.parseInt(size);
- if (nSize < 0) {
- throw new IllegalArgumentException("Quota cannot be negative.");
- }
-
- return new OzoneQuota(nSize, currUnit);
- }
-
-
- /**
- * Checks if Quota String is just as remove operation.
- *
- * @param quotaString User provided quota String
- *
- * @return True if it is Remove, false otherwise
- */
- public static boolean isRemove(String quotaString) {
-
- return (quotaString != null) &&
- (quotaString.compareToIgnoreCase(Header.OZONE_QUOTA_REMOVE) == 0);
- }
-
- /**
- * Returns size in Bytes or -1 if there is no Quota.
- */
- @JsonIgnore
- public long sizeInBytes() {
- switch (this.unit) {
- case BYTES:
- return this.getSize();
- case MB:
- return this.getSize() * OzoneConsts.MB;
- case GB:
- return this.getSize() * OzoneConsts.GB;
- case TB:
- return this.getSize() * OzoneConsts.TB;
- case UNDEFINED:
- default:
- return -1;
- }
- }
-
- /**
- * Returns OzoneQuota corresponding to size in bytes.
- *
- * @param sizeInBytes size in bytes to be converted
- *
- * @return OzoneQuota object
- */
- public static OzoneQuota getOzoneQuota(long sizeInBytes) {
- long size;
- Units unit;
- if (sizeInBytes % OzoneConsts.TB == 0) {
- size = sizeInBytes / OzoneConsts.TB;
- unit = Units.TB;
- } else if (sizeInBytes % OzoneConsts.GB == 0) {
- size = sizeInBytes / OzoneConsts.GB;
- unit = Units.GB;
- } else if (sizeInBytes % OzoneConsts.MB == 0) {
- size = sizeInBytes / OzoneConsts.MB;
- unit = Units.MB;
- } else {
- size = sizeInBytes;
- unit = Units.BYTES;
- }
- return new OzoneQuota((int)size, unit);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
deleted file mode 100644
index 4fbc18f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/request/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.request;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
deleted file mode 100644
index e66cd20..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/BucketInfo.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.base.Preconditions;
-
-/**
- * BucketInfo class, this is used as response class to send
- * Json info about a bucket back to a client.
- */
-public class BucketInfo implements Comparable<BucketInfo> {
- static final String BUCKET_INFO = "BUCKET_INFO_FILTER";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(BucketInfo.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"bytesUsed", "keyCount"};
-
- FilterProvider filters = new SimpleFilterProvider().addFilter(BUCKET_INFO,
- SimpleBeanPropertyFilter.serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- private String volumeName;
- private String bucketName;
- private String createdOn;
- private List<OzoneAcl> acls;
- private OzoneConsts.Versioning versioning;
- private StorageType storageType;
- private long bytesUsed;
- private long keyCount;
-
- /**
- * Constructor for BucketInfo.
- *
- * @param volumeName
- * @param bucketName
- */
- public BucketInfo(String volumeName, String bucketName) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- }
-
-
- /**
- * Default constructor for BucketInfo.
- */
- public BucketInfo() {
- acls = new LinkedList<OzoneAcl>();
- }
-
- /**
- * Parse a JSON string into BucketInfo Object.
- *
- * @param jsonString - Json String
- *
- * @return - BucketInfo
- *
- * @throws IOException
- */
- public static BucketInfo parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
- /**
- * Returns a List of ACL on the Bucket.
- *
- * @return List of Acls
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Sets ACls.
- *
- * @param acls - Acls list
- */
- public void setAcls(List<OzoneAcl> acls) {
- this.acls = acls;
- }
-
- /**
- * Returns Storage Type info.
- *
- * @return Storage Type of the bucket
- */
- public StorageType getStorageType() {
- return storageType;
- }
-
- /**
- * Sets the Storage Type.
- *
- * @param storageType - Storage Type
- */
- public void setStorageType(StorageType storageType) {
- this.storageType = storageType;
- }
-
- /**
- * Returns versioning.
- *
- * @return versioning Enum
- */
- public OzoneConsts.Versioning getVersioning() {
- return versioning;
- }
-
- /**
- * Sets Versioning.
- *
- * @param versioning
- */
- public void setVersioning(OzoneConsts.Versioning versioning) {
- this.versioning = versioning;
- }
-
-
- /**
- * Gets bucket Name.
- *
- * @return String
- */
- public String getBucketName() {
- return bucketName;
- }
-
- /**
- * Sets bucket Name.
- *
- * @param bucketName - Name of the bucket
- */
- public void setBucketName(String bucketName) {
- this.bucketName = bucketName;
- }
-
- /**
- * Sets creation time of the bucket.
- *
- * @param creationTime - Date String
- */
- public void setCreatedOn(String creationTime) {
- this.createdOn = creationTime;
- }
-
- /**
- * Returns creation time.
- *
- * @return creation time of bucket.
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * Returns a JSON string of this object.
- * After stripping out bytesUsed and keyCount
- *
- * @return String
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * Returns the Object as a Json String.
- *
- * The reason why both toJSONString exists and toDBString exists
- * is because toJSONString supports an external facing contract with
- * REST clients. However server internally would want to add more
- * fields to this class. The distinction helps in serializing all
- * fields vs. only fields that are part of REST protocol.
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-
- /**
- * Returns Volume Name.
- *
- * @return String volume name
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Sets the Volume Name of the bucket.
- *
- * @param volumeName - volumeName
- */
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * Please note : BucketInfo compare functions are used only within the
- * context of a volume, hence volume name is purposefully ignored in
- * compareTo, equal and hashcode functions of this class.
- */
- @Override
- public int compareTo(BucketInfo o) {
- Preconditions.checkState(o.getVolumeName().equals(this.getVolumeName()));
- return this.bucketName.compareTo(o.getBucketName());
- }
-
- /**
- * Checks if two bucketInfo's are equal.
- * @param o Object BucketInfo
- * @return True or False
- */
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (!(o instanceof BucketInfo)) {
- return false;
- }
-
- BucketInfo that = (BucketInfo) o;
- Preconditions.checkState(that.getVolumeName().equals(this.getVolumeName()));
- return bucketName.equals(that.bucketName);
-
- }
-
- /**
- * Hash Code for this object.
- * @return int
- */
- @Override
- public int hashCode() {
- return bucketName.hashCode();
- }
-
- /**
- * Get the number of bytes used by this bucket.
- *
- * @return long
- */
- public long getBytesUsed() {
- return bytesUsed;
- }
-
- /**
- * Set bytes Used.
- *
- * @param bytesUsed - bytesUsed
- */
- public void setBytesUsed(long bytesUsed) {
- this.bytesUsed = bytesUsed;
- }
-
- /**
- * Get Key Count inside this bucket.
- *
- * @return - KeyCount
- */
- public long getKeyCount() {
- return keyCount;
- }
-
- /**
- * Set Key Count inside this bucket.
- *
- * @param keyCount - Sets the Key Count
- */
- public void setKeyCount(long keyCount) {
- this.keyCount = keyCount;
- }
-
- /**
- * This class allows us to create custom filters
- * for the Json serialization.
- */
- @JsonFilter(BUCKET_INFO)
- class MixIn {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
deleted file mode 100644
index 34885f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/KeyInfo.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-
-import org.apache.commons.lang.builder.EqualsBuilder;
-import org.apache.commons.lang.builder.HashCodeBuilder;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * Represents an Ozone key Object.
- */
-public class KeyInfo implements Comparable<KeyInfo> {
- static final String OBJECT_INFO = "OBJECT_INFO_FILTER";
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(KeyInfo.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"dataFileName"};
-
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(OBJECT_INFO, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- /**
- * This class allows us to create custom filters
- * for the Json serialization.
- */
- @JsonFilter(OBJECT_INFO)
- class MixIn {
-
- }
- private long version;
- private String md5hash;
- private String createdOn;
- private String modifiedOn;
- private long size;
- private String keyName;
-
- private String dataFileName;
-
- /**
- * When this key was created.
- *
- * @return Date String
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * When this key was modified.
- *
- * @return Date String
- */
- public String getModifiedOn() {
- return modifiedOn;
- }
-
- /**
- * When this key was created.
- *
- * @param createdOn - Date String
- */
- public void setCreatedOn(String createdOn) {
- this.createdOn = createdOn;
- }
-
- /**
- * When this key was modified.
- *
- * @param modifiedOn - Date String
- */
- public void setModifiedOn(String modifiedOn) {
- this.modifiedOn = modifiedOn;
- }
-
- /**
- * Full path to where the actual data for this key is stored.
- *
- * @return String
- */
- public String getDataFileName() {
- return dataFileName;
- }
-
- /**
- * Sets up where the file path is stored.
- *
- * @param dataFileName - Data File Name
- */
- public void setDataFileName(String dataFileName) {
- this.dataFileName = dataFileName;
- }
-
- /**
- * Gets the Keyname of this object.
- *
- * @return String
- */
- public String getKeyName() {
- return keyName;
- }
-
- /**
- * Sets the Key name of this object.
- *
- * @param keyName - String
- */
- public void setKeyName(String keyName) {
- this.keyName = keyName;
- }
-
- /**
- * Returns the MD5 Hash for the data of this key.
- *
- * @return String MD5
- */
- public String getMd5hash() {
- return md5hash;
- }
-
- /**
- * Sets the MD5 of this file.
- *
- * @param md5hash - Md5 of this file
- */
- public void setMd5hash(String md5hash) {
- this.md5hash = md5hash;
- }
-
- /**
- * Number of bytes stored in the data part of this key.
- *
- * @return long size of the data file
- */
- public long getSize() {
- return size;
- }
-
- /**
- * Sets the size of the Data part of this key.
- *
- * @param size - Size in long
- */
- public void setSize(long size) {
- this.size = size;
- }
-
- /**
- * Version of this key.
- *
- * @return - returns the version of this key.
- */
- public long getVersion() {
- return version;
- }
-
- /**
- * Sets the version of this key.
- *
- * @param version - Version String
- */
- public void setVersion(long version) {
- this.version = version;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- *
- * @return a negative integer, zero, or a positive integer as this object
- * is less than, equal to, or greater than the specified object.
- *
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(KeyInfo o) {
- if (this.keyName.compareTo(o.getKeyName()) != 0) {
- return this.keyName.compareTo(o.getKeyName());
- }
-
- if (this.getVersion() == o.getVersion()) {
- return 0;
- }
- if (this.getVersion() < o.getVersion()) {
- return -1;
- }
- return 1;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
-
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- KeyInfo keyInfo = (KeyInfo) o;
-
- return new EqualsBuilder()
- .append(version, keyInfo.version)
- .append(keyName, keyInfo.keyName)
- .isEquals();
- }
-
- @Override
- public int hashCode() {
- return new HashCodeBuilder(17, 37)
- .append(version)
- .append(keyName)
- .toHashCode();
- }
-
- /**
-
- * Parse a string to retuen BucketInfo Object.
- *
- * @param jsonString - Json String
- *
- * @return - BucketInfo
- *
- * @throws IOException
- */
- public static KeyInfo parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
-
- /**
- * Returns a JSON string of this object.
- * After stripping out bytesUsed and keyCount
- *
- * @return String
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * Returns the Object as a Json String.
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
deleted file mode 100644
index bc4e65b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListBuckets.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * List Bucket is the response for the ListBucket Query.
- */
-public class ListBuckets {
- static final String BUCKET_LIST = "BUCKET_LIST_FILTER";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(ListBuckets.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"dataFileName"};
-
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(BUCKET_LIST, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- private List<BucketInfo> buckets;
-
- /**
- * Constructor for ListBuckets.
- * @param buckets - List of buckets owned by this user
- */
- public ListBuckets(List<BucketInfo> buckets) {
- this.buckets = buckets;
-
- }
-
- /**
- * Constructor for ListBuckets.
- */
- public ListBuckets() {
- this.buckets = new LinkedList<BucketInfo>();
- }
-
- /**
- * Parses a String to return ListBuckets object.
- *
- * @param data - Json String
- *
- * @return - ListBuckets
- *
- * @throws IOException
- */
- public static ListBuckets parse(String data) throws IOException {
- return READER.readValue(data);
- }
-
- /**
- * Returns a list of Buckets.
- *
- * @return Bucket list
- */
- public List<BucketInfo> getBuckets() {
- return buckets;
- }
-
- /**
- * Sets the list of buckets owned by this user.
- *
- * @param buckets - List of Buckets
- */
- public void setBuckets(List<BucketInfo> buckets) {
- this.buckets = buckets;
- }
-
-
- /**
- * Returns a JSON string of this object.
- * After stripping out bytesUsed and keyCount
- *
- * @return String
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * Returns the Object as a Json String.
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-
- /**
- * Sorts the buckets based on bucketName.
- * This is useful when we return the list of buckets
- */
- public void sort() {
- Collections.sort(buckets);
- }
-
- /**
- * Add a new bucket to the list of buckets.
- * @param bucketInfo - bucket Info
- */
- public void addBucket(BucketInfo bucketInfo){
- this.buckets.add(bucketInfo);
- }
-
- /**
- * This class allows us to create custom filters
- * for the Json serialization.
- */
- @JsonFilter(BUCKET_LIST)
- class MixIn {
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
deleted file mode 100644
index 9dc77d2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListKeys.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-import com.google.common.base.Preconditions;
-
-/**
- * This class the represents the list of keys (Objects) in a bucket.
- */
-public class ListKeys {
- static final String OBJECT_LIST = "OBJECT_LIST_FILTER";
-
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(ListKeys.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"dataFileName"};
-
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(OBJECT_LIST, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- private String name;
- private String prefix;
- private long maxKeys;
- private boolean truncated;
- private List<KeyInfo> keyList;
-
- /**
- * Default constructor needed for json serialization.
- */
- public ListKeys() {
- this.keyList = new LinkedList<>();
- }
-
- /**
- * Constructor for ListKeys.
- *
- * @param args ListArgs
- * @param truncated is truncated
- */
- public ListKeys(ListArgs args, boolean truncated) {
- Preconditions.checkState(args.getArgs() instanceof BucketArgs);
- this.name = ((BucketArgs) args.getArgs()).getBucketName();
- this.prefix = args.getPrefix();
- this.maxKeys = args.getMaxKeys();
- this.truncated = truncated;
- }
-
- /**
- * Converts a Json string to POJO.
- * @param jsonString - json string.
- * @return ListObject
- * @throws IOException - Json conversion error.
- */
- public static ListKeys parse(String jsonString) throws IOException {
- return READER.readValue(jsonString);
- }
-
- /**
- * Returns a list of Objects.
- *
- * @return List of KeyInfo Objects.
- */
- public List<KeyInfo> getKeyList() {
- return keyList;
- }
-
- /**
- * Sets the list of Objects.
- *
- * @param objectList - List of Keys
- */
- public void setKeyList(List<KeyInfo> objectList) {
- this.keyList = objectList;
- }
-
- /**
- * Gets the Max Key Count.
- *
- * @return long
- */
- public long getMaxKeys() {
- return maxKeys;
- }
-
- /**
- * Gets bucket Name.
- *
- * @return String
- */
- public String getName() {
- return name;
- }
-
- /**
- * Gets Prefix.
- *
- * @return String
- */
- public String getPrefix() {
- return prefix;
- }
-
- /**
- * Gets truncated Status.
- *
- * @return Boolean
- */
- public boolean isTruncated() {
- return truncated;
- }
-
- /**
- * Sets the value of truncated.
- *
- * @param value - Boolean
- */
- public void setTruncated(boolean value) {
- this.truncated = value;
- }
-
- /**
- * Returns a JSON string of this object. After stripping out bytesUsed and
- * keyCount.
- *
- * @return String
- * @throws IOException - On json Errors.
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * Returns the Object as a Json String.
- *
- * @return String
- * @throws IOException - on json errors.
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-
- /**
- * Sorts the keys based on name and version. This is useful when we return the
- * list of keys.
- */
- public void sort() {
- Collections.sort(keyList);
- }
-
- /**
- * Add a new key to the list of keys.
- * @param keyInfo - key Info
- */
- public void addKey(KeyInfo keyInfo){
- this.keyList.add(keyInfo);
- }
-
- /**
- * This class allows us to create custom filters for the Json serialization.
- */
- @JsonFilter(OBJECT_LIST)
- class MixIn {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
deleted file mode 100644
index b918349..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/ListVolumes.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.response;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * List Volume Class is the class that is returned in JSON format to
- * users when they call ListVolumes.
- */
-@InterfaceAudience.Private
-public class ListVolumes {
- private List<VolumeInfo> volumes;
-
- static final String VOLUME_LIST = "VOLUME_LIST_FILTER";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(ListVolumes.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"bytesUsed", "bucketCount"};
-
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(VOLUME_LIST, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- /**
- * Used for json filtering.
- */
- @JsonFilter(VOLUME_LIST)
- class MixIn {
- }
-
- /**
- * Constructs ListVolume objects.
- */
- public ListVolumes() {
- this.volumes = new LinkedList<VolumeInfo>();
- }
-
- /**
- * Gets the list of volumes.
- *
- * @return List of VolumeInfo Objects
- */
- public List<VolumeInfo> getVolumes() {
- return volumes;
- }
-
-
- /**
- * Sets volume info.
- *
- * @param volumes - List of Volumes
- */
- public void setVolumes(List<VolumeInfo> volumes) {
- this.volumes = volumes;
- }
-
- /**
- * Returns a JSON string of this object.
- * After stripping out bytesUsed and bucketCount
- *
- * @return String
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * When we serialize a volumeInfo to our database
- * we will use all fields. However the toJsonString
- * will strip out bytesUsed and bucketCount from the
- * volume Info
- *
- * @return Json String
- *
- * @throws IOException
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-
- /**
- * Parses a String to return ListVolumes object.
- *
- * @param data - Json String
- *
- * @return - ListVolumes
- *
- * @throws IOException
- */
- public static ListVolumes parse(String data) throws IOException {
- return READER.readValue(data);
- }
-
- /**
- * Adds a new volume info to the List.
- *
- * @param info - VolumeInfo
- */
- public void addVolume(VolumeInfo info) {
- this.volumes.add(info);
- }
-
- /**
- * Sorts the volume names based on volume name.
- * This is useful when we return the list of volume names
- */
- public void sort() {
- Collections.sort(volumes);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
deleted file mode 100644
index 112b27e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeInfo.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.response;
-
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonFilter;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.ser.FilterProvider;
-import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
-import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;
-
-/**
- * VolumeInfo Class is the Java class that represents
- * Json when VolumeInfo Call is made.
- */
-@InterfaceAudience.Private
-public class VolumeInfo implements Comparable<VolumeInfo> {
-
- static final String VOLUME_INFO = "VOLUME_INFO_FILTER";
- private static final ObjectReader READER =
- new ObjectMapper().readerFor(VolumeInfo.class);
- private static final ObjectWriter WRITER;
-
- static {
- ObjectMapper mapper = new ObjectMapper();
- String[] ignorableFieldNames = {"bytesUsed", "bucketCount"};
-
- FilterProvider filters = new SimpleFilterProvider()
- .addFilter(VOLUME_INFO, SimpleBeanPropertyFilter
- .serializeAllExcept(ignorableFieldNames));
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- mapper.addMixIn(Object.class, MixIn.class);
-
- mapper.setFilterProvider(filters);
- WRITER = mapper.writerWithDefaultPrettyPrinter();
- }
-
- /**
- * Custom Json Filter Class.
- */
- @JsonFilter(VOLUME_INFO)
- class MixIn {
- }
- private VolumeOwner owner;
- private OzoneQuota quota;
- private String volumeName;
- private String createdOn;
- private String createdBy;
-
- private long bytesUsed;
- private long bucketCount;
-
-
- /**
- * Constructor for VolumeInfo.
- *
- * @param volumeName - Name of the Volume
- * @param createdOn _ Date String
- * @param createdBy - Person who created it
- */
- public VolumeInfo(String volumeName, String createdOn, String createdBy) {
- this.createdOn = createdOn;
- this.volumeName = volumeName;
- this.createdBy = createdBy;
- }
-
- /**
- * Constructor for VolumeInfo.
- */
- public VolumeInfo() {
- }
-
- /**
- * Returns the name of the person who created this volume.
- *
- * @return Name of Admin who created this
- */
- public String getCreatedBy() {
- return createdBy;
- }
-
- /**
- * Sets the user name of the person who created this volume.
- *
- * @param createdBy - UserName
- */
- public void setCreatedBy(String createdBy) {
- this.createdBy = createdBy;
- }
-
- /**
- * Gets the date on which this volume was created.
- *
- * @return - Date String
- */
- public String getCreatedOn() {
- return createdOn;
- }
-
- /**
- * Sets the date string.
- *
- * @param createdOn - Date String
- */
- public void setCreatedOn(String createdOn) {
- this.createdOn = createdOn;
- }
-
- /**
- * Returns the owner info.
- *
- * @return - OwnerInfo
- */
- public VolumeOwner getOwner() {
- return owner;
- }
-
- /**
- * Sets the owner.
- *
- * @param owner - OwnerInfo
- */
- public void setOwner(VolumeOwner owner) {
- this.owner = owner;
- }
-
- /**
- * Returns the quota information on a volume.
- *
- * @return Quota
- */
- public OzoneQuota getQuota() {
- return quota;
- }
-
- /**
- * Sets the quota info.
- *
- * @param quota - Quota Info
- */
- public void setQuota(OzoneQuota quota) {
- this.quota = quota;
- }
-
- /**
- * gets the volume name.
- *
- * @return - Volume Name
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Sets the volume name.
- *
- * @param volumeName - Volume Name
- */
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
- /**
- * Returns a JSON string of this object.
- * After stripping out bytesUsed and bucketCount
- *
- * @return String - json string
- * @throws IOException
- */
- public String toJsonString() throws IOException {
- return WRITER.writeValueAsString(this);
- }
-
- /**
- * When we serialize a volumeInfo to our database
- * we will use all fields. However the toJsonString
- * will strip out bytesUsed and bucketCount from the
- * volume Info
- *
- * @return Json String
- *
- * @throws IOException
- */
- public String toDBString() throws IOException {
- return JsonUtils.toJsonString(this);
- }
-
-
- /**
- * Comparable Interface.
- * @param o VolumeInfo Object.
- * @return Result of comparison
- */
- @Override
- public int compareTo(VolumeInfo o) {
- return this.volumeName.compareTo(o.getVolumeName());
- }
-
- /**
- * Gets the number of bytesUsed by this volume.
- *
- * @return long - Bytes used
- */
- public long getBytesUsed() {
- return bytesUsed;
- }
-
- /**
- * Sets number of bytesUsed by this volume.
- *
- * @param bytesUsed - Number of bytesUsed
- */
- public void setBytesUsed(long bytesUsed) {
- this.bytesUsed = bytesUsed;
- }
-
- /**
- * Returns VolumeInfo class from json string.
- *
- * @param data - Json String
- *
- * @return VolumeInfo
- *
- * @throws IOException
- */
- public static VolumeInfo parse(String data) throws IOException {
- return READER.readValue(data);
- }
-
- /**
- * Indicates whether some other object is "equal to" this one.
- *
- * @param obj the reference object with which to compare.
- *
- * @return {@code true} if this object is the same as the obj
- * argument; {@code false} otherwise.
- */
- @Override
- public boolean equals(Object obj) {
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- VolumeInfo otherInfo = (VolumeInfo) obj;
- return otherInfo.getVolumeName().equals(this.getVolumeName());
- }
-
- /**
- * Returns a hash code value for the object. This method is
- * supported for the benefit of hash tables such as those provided by
- * HashMap.
- * @return a hash code value for this object.
- *
- * @see Object#equals(Object)
- * @see System#identityHashCode
- */
- @Override
- public int hashCode() {
- return getVolumeName().hashCode();
- }
-
- /**
- * Total number of buckets under this volume.
- *
- * @return - bucketCount
- */
- public long getBucketCount() {
- return bucketCount;
- }
-
- /**
- * Sets the buckets count.
- *
- * @param bucketCount - Bucket Count
- */
- public void setBucketCount(long bucketCount) {
- this.bucketCount = bucketCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
deleted file mode 100644
index afb0460..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/VolumeOwner.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.response;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Volume Owner represents the owner of a volume.
- *
- * This is a class instead of a string since we might need to extend this class
- * to support other forms of authentication.
- */
-@InterfaceAudience.Private
-public class VolumeOwner {
- @JsonInclude(JsonInclude.Include.NON_NULL)
- private String name;
-
- /**
- * Constructor for VolumeOwner.
- *
- * @param name - name of the User
- */
- public VolumeOwner(String name) {
- this.name = name;
- }
-
- /**
- * Constructs Volume Owner.
- */
- public VolumeOwner() {
- name = null;
- }
-
- /**
- * Returns the user name.
- *
- * @return Name
- */
- public String getName() {
- return name;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
deleted file mode 100644
index 3bf66c8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/response/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.response;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
deleted file mode 100644
index 1830c71..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/DistributedStorageHandler.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.storage;
-
-import com.google.common.base.Strings;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.protocolPB
- .KeySpaceManagerProtocolClientSideTranslatorPB;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.OzoneConsts.Versioning;
-import org.apache.hadoop.ozone.client.io.ChunkGroupInputStream;
-import org.apache.hadoop.ozone.client.io.ChunkGroupOutputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocolPB.KSMPBHelper;
-import org.apache.hadoop.ozone.ksm.KSMConfigKeys;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.response.VolumeOwner;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.List;
-
-/**
- * A {@link StorageHandler} implementation that distributes object storage
- * across the nodes of an HDFS cluster.
- */
-public final class DistributedStorageHandler implements StorageHandler {
- private static final Logger LOG =
- LoggerFactory.getLogger(DistributedStorageHandler.class);
-
- private final StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient;
- private final KeySpaceManagerProtocolClientSideTranslatorPB
- keySpaceManagerClient;
- private final XceiverClientManager xceiverClientManager;
- private final OzoneAcl.OzoneACLRights userRights;
- private final OzoneAcl.OzoneACLRights groupRights;
- private int chunkSize;
- private final boolean useRatis;
- private final OzoneProtos.ReplicationType type;
- private final OzoneProtos.ReplicationFactor factor;
-
- /**
- * Creates a new DistributedStorageHandler.
- *
- * @param conf configuration
- * @param storageContainerLocation StorageContainerLocationProtocol proxy
- * @param keySpaceManagerClient KeySpaceManager proxy
- */
- public DistributedStorageHandler(OzoneConfiguration conf,
- StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocation,
- KeySpaceManagerProtocolClientSideTranslatorPB
- keySpaceManagerClient) {
- this.keySpaceManagerClient = keySpaceManagerClient;
- this.storageContainerLocationClient = storageContainerLocation;
- this.xceiverClientManager = new XceiverClientManager(conf);
- this.useRatis = conf.getBoolean(
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
-
- if(useRatis) {
- type = OzoneProtos.ReplicationType.RATIS;
- factor = OzoneProtos.ReplicationFactor.THREE;
- } else {
- type = OzoneProtos.ReplicationType.STAND_ALONE;
- factor = OzoneProtos.ReplicationFactor.ONE;
- }
-
- chunkSize = conf.getInt(ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_KEY,
- ScmConfigKeys.OZONE_SCM_CHUNK_SIZE_DEFAULT);
- userRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_USER_RIGHTS,
- KSMConfigKeys.OZONE_KSM_USER_RIGHTS_DEFAULT);
- groupRights = conf.getEnum(KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS,
- KSMConfigKeys.OZONE_KSM_GROUP_RIGHTS_DEFAULT);
- if(chunkSize > ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE) {
- LOG.warn("The chunk size ({}) is not allowed to be more than"
- + " the maximum size ({}),"
- + " resetting to the maximum size.",
- chunkSize, ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE);
- chunkSize = ScmConfigKeys.OZONE_SCM_CHUNK_MAX_SIZE;
- }
- }
-
- @Override
- public void createVolume(VolumeArgs args) throws IOException, OzoneException {
- long quota = args.getQuota() == null ?
- OzoneConsts.MAX_QUOTA_IN_BYTES : args.getQuota().sizeInBytes();
- OzoneAcl userAcl =
- new OzoneAcl(OzoneAcl.OzoneACLType.USER,
- args.getUserName(), userRights);
- KsmVolumeArgs.Builder builder = KsmVolumeArgs.newBuilder();
- builder.setAdminName(args.getAdminName())
- .setOwnerName(args.getUserName())
- .setVolume(args.getVolumeName())
- .setQuotaInBytes(quota)
- .addOzoneAcls(KSMPBHelper.convertOzoneAcl(userAcl));
- if (args.getGroups() != null) {
- for (String group : args.getGroups()) {
- OzoneAcl groupAcl =
- new OzoneAcl(OzoneAcl.OzoneACLType.GROUP, group, groupRights);
- builder.addOzoneAcls(KSMPBHelper.convertOzoneAcl(groupAcl));
- }
- }
- keySpaceManagerClient.createVolume(builder.build());
- }
-
- @Override
- public void setVolumeOwner(VolumeArgs args) throws
- IOException, OzoneException {
- keySpaceManagerClient.setOwner(args.getVolumeName(), args.getUserName());
- }
-
- @Override
- public void setVolumeQuota(VolumeArgs args, boolean remove)
- throws IOException, OzoneException {
- long quota = remove ? OzoneConsts.MAX_QUOTA_IN_BYTES :
- args.getQuota().sizeInBytes();
- keySpaceManagerClient.setQuota(args.getVolumeName(), quota);
- }
-
- @Override
- public boolean checkVolumeAccess(String volume, OzoneAcl acl)
- throws IOException, OzoneException {
- return keySpaceManagerClient
- .checkVolumeAccess(volume, KSMPBHelper.convertOzoneAcl(acl));
- }
-
- @Override
- public ListVolumes listVolumes(ListArgs args)
- throws IOException, OzoneException {
- int maxNumOfKeys = args.getMaxKeys();
- if (maxNumOfKeys <= 0 ||
- maxNumOfKeys > OzoneConsts.MAX_LISTVOLUMES_SIZE) {
- throw new IllegalArgumentException(
- String.format("Illegal max number of keys specified,"
- + " the value must be in range (0, %d], actual : %d.",
- OzoneConsts.MAX_LISTVOLUMES_SIZE, maxNumOfKeys));
- }
-
- List<KsmVolumeArgs> listResult;
- if (args.isRootScan()) {
- listResult = keySpaceManagerClient.listAllVolumes(args.getPrefix(),
- args.getPrevKey(), args.getMaxKeys());
- } else {
- UserArgs userArgs = args.getArgs();
- if (userArgs == null || userArgs.getUserName() == null) {
- throw new IllegalArgumentException("Illegal argument,"
- + " missing user argument.");
- }
- listResult = keySpaceManagerClient.listVolumeByUser(
- args.getArgs().getUserName(), args.getPrefix(), args.getPrevKey(),
- args.getMaxKeys());
- }
-
- // TODO Add missing fields createdBy, bucketCount and bytesUsed
- ListVolumes result = new ListVolumes();
- for (KsmVolumeArgs volumeArgs : listResult) {
- VolumeInfo info = new VolumeInfo();
- KeySpaceManagerProtocolProtos.VolumeInfo
- infoProto = volumeArgs.getProtobuf();
- info.setOwner(new VolumeOwner(infoProto.getOwnerName()));
- info.setQuota(OzoneQuota.getOzoneQuota(infoProto.getQuotaInBytes()));
- info.setVolumeName(infoProto.getVolume());
- info.setCreatedOn(OzoneUtils.formatTime(infoProto.getCreationTime()));
- result.addVolume(info);
- }
-
- return result;
- }
-
- @Override
- public void deleteVolume(VolumeArgs args)
- throws IOException, OzoneException {
- keySpaceManagerClient.deleteVolume(args.getVolumeName());
- }
-
- @Override
- public VolumeInfo getVolumeInfo(VolumeArgs args)
- throws IOException, OzoneException {
- KsmVolumeArgs volumeArgs =
- keySpaceManagerClient.getVolumeInfo(args.getVolumeName());
- //TODO: add support for createdOn and other fields in getVolumeInfo
- VolumeInfo volInfo =
- new VolumeInfo(volumeArgs.getVolume(), null,
- volumeArgs.getAdminName());
- volInfo.setOwner(new VolumeOwner(volumeArgs.getOwnerName()));
- volInfo.setQuota(OzoneQuota.getOzoneQuota(volumeArgs.getQuotaInBytes()));
- volInfo.setCreatedOn(OzoneUtils.formatTime(volumeArgs.getCreationTime()));
- return volInfo;
- }
-
- @Override
- public void createBucket(final BucketArgs args)
- throws IOException, OzoneException {
- KsmBucketInfo.Builder builder = KsmBucketInfo.newBuilder();
- builder.setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName());
- if(args.getAddAcls() != null) {
- builder.setAcls(args.getAddAcls());
- }
- if(args.getStorageType() != null) {
- builder.setStorageType(args.getStorageType());
- }
- if(args.getVersioning() != null) {
- builder.setIsVersionEnabled(getBucketVersioningProtobuf(
- args.getVersioning()));
- }
- keySpaceManagerClient.createBucket(builder.build());
- }
-
- /**
- * Converts OzoneConts.Versioning enum to boolean.
- *
- * @param version
- * @return corresponding boolean value
- */
- private boolean getBucketVersioningProtobuf(
- Versioning version) {
- if(version != null) {
- switch(version) {
- case ENABLED:
- return true;
- case NOT_DEFINED:
- case DISABLED:
- default:
- return false;
- }
- }
- return false;
- }
-
- @Override
- public void setBucketAcls(BucketArgs args)
- throws IOException, OzoneException {
- List<OzoneAcl> removeAcls = args.getRemoveAcls();
- List<OzoneAcl> addAcls = args.getAddAcls();
- if(removeAcls != null || addAcls != null) {
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName());
- if(removeAcls != null && !removeAcls.isEmpty()) {
- builder.setRemoveAcls(args.getRemoveAcls());
- }
- if(addAcls != null && !addAcls.isEmpty()) {
- builder.setAddAcls(args.getAddAcls());
- }
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
- }
-
- @Override
- public void setBucketVersioning(BucketArgs args)
- throws IOException, OzoneException {
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setIsVersionEnabled(getBucketVersioningProtobuf(
- args.getVersioning()));
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void setBucketStorageClass(BucketArgs args)
- throws IOException, OzoneException {
- KsmBucketArgs.Builder builder = KsmBucketArgs.newBuilder();
- builder.setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setStorageType(args.getStorageType());
- keySpaceManagerClient.setBucketProperty(builder.build());
- }
-
- @Override
- public void deleteBucket(BucketArgs args)
- throws IOException, OzoneException {
- keySpaceManagerClient.deleteBucket(args.getVolumeName(),
- args.getBucketName());
- }
-
- @Override
- public void checkBucketAccess(BucketArgs args)
- throws IOException, OzoneException {
- throw new UnsupportedOperationException(
- "checkBucketAccess not implemented");
- }
-
- @Override
- public ListBuckets listBuckets(ListArgs args)
- throws IOException, OzoneException {
- ListBuckets result = new ListBuckets();
- UserArgs userArgs = args.getArgs();
- if (userArgs instanceof VolumeArgs) {
- VolumeArgs va = (VolumeArgs) userArgs;
- if (Strings.isNullOrEmpty(va.getVolumeName())) {
- throw new IllegalArgumentException("Illegal argument,"
- + " volume name cannot be null or empty.");
- }
-
- int maxNumOfKeys = args.getMaxKeys();
- if (maxNumOfKeys <= 0 ||
- maxNumOfKeys > OzoneConsts.MAX_LISTBUCKETS_SIZE) {
- throw new IllegalArgumentException(
- String.format("Illegal max number of keys specified,"
- + " the value must be in range (0, %d], actual : %d.",
- OzoneConsts.MAX_LISTBUCKETS_SIZE, maxNumOfKeys));
- }
-
- List<KsmBucketInfo> buckets =
- keySpaceManagerClient.listBuckets(va.getVolumeName(),
- args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
-
- // Convert the result for the web layer.
- for (KsmBucketInfo bucketInfo : buckets) {
- BucketInfo bk = new BucketInfo();
- bk.setVolumeName(bucketInfo.getVolumeName());
- bk.setBucketName(bucketInfo.getBucketName());
- bk.setStorageType(bucketInfo.getStorageType());
- bk.setAcls(bucketInfo.getAcls());
- bk.setCreatedOn(OzoneUtils.formatTime(bucketInfo.getCreationTime()));
- result.addBucket(bk);
- }
- return result;
- } else {
- throw new IllegalArgumentException("Illegal argument provided,"
- + " expecting VolumeArgs type but met "
- + userArgs.getClass().getSimpleName());
- }
- }
-
- @Override
- public BucketInfo getBucketInfo(BucketArgs args)
- throws IOException {
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- KsmBucketInfo ksmBucketInfo = keySpaceManagerClient.getBucketInfo(
- volumeName, bucketName);
- BucketInfo bucketInfo = new BucketInfo(ksmBucketInfo.getVolumeName(),
- ksmBucketInfo.getBucketName());
- if(ksmBucketInfo.getIsVersionEnabled()) {
- bucketInfo.setVersioning(Versioning.ENABLED);
- } else {
- bucketInfo.setVersioning(Versioning.DISABLED);
- }
- bucketInfo.setStorageType(ksmBucketInfo.getStorageType());
- bucketInfo.setAcls(ksmBucketInfo.getAcls());
- bucketInfo.setCreatedOn(
- OzoneUtils.formatTime(ksmBucketInfo.getCreationTime()));
- return bucketInfo;
- }
-
- @Override
- public OutputStream newKeyWriter(KeyArgs args) throws IOException,
- OzoneException {
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setDataSize(args.getSize())
- .setType(xceiverClientManager.getType())
- .setFactor(xceiverClientManager.getFactor())
- .build();
- // contact KSM to allocate a block for key.
- OpenKeySession openKey = keySpaceManagerClient.openKey(keyArgs);
- ChunkGroupOutputStream groupOutputStream =
- new ChunkGroupOutputStream.Builder()
- .setHandler(openKey)
- .setXceiverClientManager(xceiverClientManager)
- .setScmClient(storageContainerLocationClient)
- .setKsmClient(keySpaceManagerClient)
- .setChunkSize(chunkSize)
- .setRequestID(args.getRequestID())
- .setType(xceiverClientManager.getType())
- .setFactor(xceiverClientManager.getFactor())
- .build();
- groupOutputStream.addPreallocateBlocks(
- openKey.getKeyInfo().getLatestVersionLocations(),
- openKey.getOpenVersion());
- return new OzoneOutputStream(groupOutputStream);
- }
-
- @Override
- public void commitKey(KeyArgs args, OutputStream stream) throws
- IOException, OzoneException {
- stream.close();
- }
-
- @Override
- public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
- OzoneException {
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .setDataSize(args.getSize())
- .build();
- KsmKeyInfo keyInfo = keySpaceManagerClient.lookupKey(keyArgs);
- return ChunkGroupInputStream.getFromKsmKeyInfo(
- keyInfo, xceiverClientManager, storageContainerLocationClient,
- args.getRequestID());
- }
-
- @Override
- public void deleteKey(KeyArgs args) throws IOException, OzoneException {
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .build();
- keySpaceManagerClient.deleteKey(keyArgs);
- }
-
- @Override
- public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
- KsmKeyArgs keyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(args.getVolumeName())
- .setBucketName(args.getBucketName())
- .setKeyName(args.getKeyName())
- .build();
-
- KsmKeyInfo ksmKeyInfo = keySpaceManagerClient.lookupKey(keyArgs);
- KeyInfo keyInfo = new KeyInfo();
- keyInfo.setVersion(0);
- keyInfo.setKeyName(ksmKeyInfo.getKeyName());
- keyInfo.setSize(ksmKeyInfo.getDataSize());
- keyInfo.setCreatedOn(
- OzoneUtils.formatTime(ksmKeyInfo.getCreationTime()));
- keyInfo.setModifiedOn(
- OzoneUtils.formatTime(ksmKeyInfo.getModificationTime()));
- return keyInfo;
- }
-
- @Override
- public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
- ListKeys result = new ListKeys();
- UserArgs userArgs = args.getArgs();
- if (userArgs instanceof BucketArgs) {
- BucketArgs bucketArgs = (BucketArgs) userArgs;
- if (Strings.isNullOrEmpty(bucketArgs.getVolumeName())) {
- throw new IllegalArgumentException("Illegal argument,"
- + " volume name cannot be null or empty.");
- }
-
- if (Strings.isNullOrEmpty(bucketArgs.getBucketName())) {
- throw new IllegalArgumentException("Illegal argument,"
- + " bucket name cannot be null or empty.");
- }
-
- int maxNumOfKeys = args.getMaxKeys();
- if (maxNumOfKeys <= 0 ||
- maxNumOfKeys > OzoneConsts.MAX_LISTKEYS_SIZE) {
- throw new IllegalArgumentException(
- String.format("Illegal max number of keys specified,"
- + " the value must be in range (0, %d], actual : %d.",
- OzoneConsts.MAX_LISTKEYS_SIZE, maxNumOfKeys));
- }
-
- List<KsmKeyInfo> keys=
- keySpaceManagerClient.listKeys(bucketArgs.getVolumeName(),
- bucketArgs.getBucketName(),
- args.getPrevKey(), args.getPrefix(), args.getMaxKeys());
-
- // Convert the result for the web layer.
- for (KsmKeyInfo info : keys) {
- KeyInfo tempInfo = new KeyInfo();
- tempInfo.setVersion(0);
- tempInfo.setKeyName(info.getKeyName());
- tempInfo.setSize(info.getDataSize());
- tempInfo.setCreatedOn(
- OzoneUtils.formatTime(info.getCreationTime()));
- tempInfo.setModifiedOn(
- OzoneUtils.formatTime(info.getModificationTime()));
-
- result.addKey(tempInfo);
- }
- return result;
- } else {
- throw new IllegalArgumentException("Illegal argument provided,"
- + " expecting BucketArgs type but met "
- + userArgs.getClass().getSimpleName());
- }
- }
-
- /**
- * Closes DistributedStorageHandler.
- */
- @Override
- public void close() {
- IOUtils.cleanupWithLogger(LOG, xceiverClientManager);
- IOUtils.cleanupWithLogger(LOG, keySpaceManagerClient);
- IOUtils.cleanupWithLogger(LOG, storageContainerLocationClient);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
deleted file mode 100644
index f5499f5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/storage/package-info.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Ozone storage handler implementation integrating REST interface front-end
- * with container data pipeline back-end.
- */
-@InterfaceAudience.Private
-package org.apache.hadoop.ozone.web.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
deleted file mode 100644
index 397c80f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/Simple.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.userauth;
-
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import javax.ws.rs.core.HttpHeaders;
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Simple is an UserAuth class that is used in the insecure
- * mode of ozone. This maps more or less to the simple user scheme in
- * HDFS.
- */
-@InterfaceAudience.Private
-public class Simple implements UserAuth {
- /**
- * Returns the x-ozone-user or the user on behalf of, This is
- * used in volume creation path.
- *
- * @param userArgs - UserArgs
- *
- * @throws OzoneException
- */
- @Override
- public String getOzoneUser(UserArgs userArgs) throws OzoneException {
- assert userArgs != null : "userArgs cannot be null";
-
- HttpHeaders headers = userArgs.getHeaders();
- List<String> users = headers.getRequestHeader(Header.OZONE_USER);
-
- if ((users == null) || (users.size() == 0)) {
- return null;
- }
- if (users.size() > 1) {
- throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
- }
- return users.get(0).toLowerCase().trim();
- }
-
- /**
- * Returns the user name as a string from the URI and HTTP headers.
- *
- * @param userArgs - user args
- *
- * @throws OzoneException -- Allows the underlying system
- * to throw, that error will get propagated to clients
- */
- @Override
- public String getUser(UserArgs userArgs) throws OzoneException {
- assert userArgs != null : "userArgs cannot be null";
-
- HttpHeaders headers = userArgs.getHeaders();
- List<String> users = headers.getRequestHeader(HttpHeaders.AUTHORIZATION);
- if (users == null || users.size() > 1) {
- throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
- }
-
- if (users.size() == 0) {
- return null;
- }
-
- String user = users.get(0).trim();
- if (user.startsWith(Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME)) {
- user = user.replace(Header.OZONE_SIMPLE_AUTHENTICATION_SCHEME, "");
- return user.toLowerCase().trim();
- } else {
- throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
- }
- }
-
-
- /**
- * Returns true if a user is a Admin - {root and hdfs are treated as admins}.
- *
- * @param userArgs - User Args
- *
- * @throws OzoneException -- Allows the underlying system
- * to throw, that error will get propagated to clients
- */
- @Override
- public boolean isAdmin(UserArgs userArgs) throws OzoneException {
- assert userArgs != null : "userArgs cannot be null";
-
- String user;
- String currentUser;
- try {
- user = getUser(userArgs);
- currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- throw ErrorTable.newError(ErrorTable.BAD_AUTHORIZATION, userArgs);
- }
- return
- (user.compareToIgnoreCase(OzoneConsts.OZONE_SIMPLE_ROOT_USER) == 0) ||
- (user.compareToIgnoreCase(OzoneConsts.OZONE_SIMPLE_HDFS_USER) == 0)
- || (user.compareToIgnoreCase(currentUser) == 0);
- }
-
- /**
- * Returns true if the request is Anonymous.
- *
- * @param userArgs - user Args
- *
- * @throws OzoneException -- Allows the underlying system
- * to throw, that error will get propagated to clients
- */
- @Override
- public boolean isAnonymous(UserArgs userArgs) throws OzoneException {
- assert userArgs != null : "userArgs cannot be null";
-
- return getUser(userArgs) == null;
- }
-
- /**
- * Returns true if the name is a recognizable user in the system.
- *
- * @param userName - Name of the user
- * @param userArgs - user Args
- *
- * @throws OzoneException -- Allows the underlying system
- * to throw, that error will get propagated to clients
- */
- @Override
- public boolean isUser(String userName, UserArgs userArgs)
- throws OzoneException {
- // In the simple case, all non-null users names are users :)
- return userName != null;
- }
-
- /**
- * Returns all the Groups that user is a member of.
- *
- * @param userArgs - User Args
- *
- * @return String Array which contains 0 or more group names
- *
- * @throws OzoneException
- */
- @Override
- public String[] getGroups(UserArgs userArgs) throws OzoneException {
- // Not implemented
- return null;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
deleted file mode 100644
index d498fc8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/userauth/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-/**
- * Netty-based HTTP server implementation for Ozone.
- */
-package org.apache.hadoop.ozone.web.userauth;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
deleted file mode 100644
index 909873f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/utils/JsonUtils.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.utils;
-
-import java.io.IOException;
-import java.util.List;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectReader;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.fasterxml.jackson.databind.type.CollectionType;
-
-/**
- * JSON Utility functions used in ozone.
- */
-public final class JsonUtils {
-
- // Reuse ObjectMapper instance for improving performance.
- // ObjectMapper is thread safe as long as we always configure instance
- // before use.
- private static final ObjectMapper MAPPER = new ObjectMapper();
- private static final ObjectReader READER = MAPPER.readerFor(Object.class);
- private static final ObjectWriter WRITTER =
- MAPPER.writerWithDefaultPrettyPrinter();
-
- private JsonUtils() {
- // Never constructed
- }
-
- public static String toJsonStringWithDefaultPrettyPrinter(String jsonString)
- throws IOException {
- Object json = READER.readValue(jsonString);
- return WRITTER.writeValueAsString(json);
- }
-
- public static String toJsonString(Object obj) throws IOException {
- return MAPPER.writeValueAsString(obj);
- }
-
- /**
- * Deserialize a list of elements from a given string,
- * each element in the list is in the given type.
- *
- * @param str json string.
- * @param elementType element type.
- * @return List of elements of type elementType
- * @throws IOException
- */
- public static List<?> toJsonList(String str, Class<?> elementType)
- throws IOException {
- CollectionType type = MAPPER.getTypeFactory()
- .constructCollectionType(List.class, elementType);
- return MAPPER.readValue(str, type);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
deleted file mode 100644
index c4bf158..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/angular-1.6.4.min.js
+++ /dev/null
@@ -1,332 +0,0 @@
-/*
- AngularJS v1.6.4
- (c) 2010-2017 Google, Inc. http://angularjs.org
- License: MIT
-*/
-(function(x){'use strict';function L(a,b){b=b||Error;return function(){var d=arguments[0],c;c="["+(a?a+":":"")+d+"] http://errors.angularjs.org/1.6.4/"+(a?a+"/":"")+d;for(d=1;d<arguments.length;d++){c=c+(1==d?"?":"&")+"p"+(d-1)+"=";var e=encodeURIComponent,f;f=arguments[d];f="function"==typeof f?f.toString().replace(/ \{[\s\S]*$/,""):"undefined"==typeof f?"undefined":"string"!=typeof f?JSON.stringify(f):f;c+=e(f)}return new b(c)}}function me(a){if(C(a))u(a.objectMaxDepth)&&(Ic.objectMaxDepth=Sb(a.objectMaxDepth)?
-a.objectMaxDepth:NaN);else return Ic}function Sb(a){return ba(a)&&0<a}function qa(a){if(null==a||Wa(a))return!1;if(H(a)||F(a)||B&&a instanceof B)return!0;var b="length"in Object(a)&&a.length;return ba(b)&&(0<=b&&(b-1 in a||a instanceof Array)||"function"===typeof a.item)}function q(a,b,d){var c,e;if(a)if(D(a))for(c in a)"prototype"!==c&&"length"!==c&&"name"!==c&&a.hasOwnProperty(c)&&b.call(d,a[c],c,a);else if(H(a)||qa(a)){var f="object"!==typeof a;c=0;for(e=a.length;c<e;c++)(f||c in a)&&b.call(d,
-a[c],c,a)}else if(a.forEach&&a.forEach!==q)a.forEach(b,d,a);else if(Jc(a))for(c in a)b.call(d,a[c],c,a);else if("function"===typeof a.hasOwnProperty)for(c in a)a.hasOwnProperty(c)&&b.call(d,a[c],c,a);else for(c in a)ua.call(a,c)&&b.call(d,a[c],c,a);return a}function Kc(a,b,d){for(var c=Object.keys(a).sort(),e=0;e<c.length;e++)b.call(d,a[c[e]],c[e]);return c}function Lc(a){return function(b,d){a(d,b)}}function ne(){return++qb}function Tb(a,b,d){for(var c=a.$$hashKey,e=0,f=b.length;e<f;++e){var g=b[e];
-if(C(g)||D(g))for(var h=Object.keys(g),k=0,l=h.length;k<l;k++){var m=h[k],n=g[m];d&&C(n)?ga(n)?a[m]=new Date(n.valueOf()):Xa(n)?a[m]=new RegExp(n):n.nodeName?a[m]=n.cloneNode(!0):Ub(n)?a[m]=n.clone():(C(a[m])||(a[m]=H(n)?[]:{}),Tb(a[m],[n],!0)):a[m]=n}}c?a.$$hashKey=c:delete a.$$hashKey;return a}function S(a){return Tb(a,va.call(arguments,1),!1)}function oe(a){return Tb(a,va.call(arguments,1),!0)}function Z(a){return parseInt(a,10)}function Vb(a,b){return S(Object.create(a),b)}function z(){}function Ya(a){return a}
-function la(a){return function(){return a}}function Wb(a){return D(a.toString)&&a.toString!==ma}function w(a){return"undefined"===typeof a}function u(a){return"undefined"!==typeof a}function C(a){return null!==a&&"object"===typeof a}function Jc(a){return null!==a&&"object"===typeof a&&!Mc(a)}function F(a){return"string"===typeof a}function ba(a){return"number"===typeof a}function ga(a){return"[object Date]"===ma.call(a)}function D(a){return"function"===typeof a}function Xa(a){return"[object RegExp]"===
-ma.call(a)}function Wa(a){return a&&a.window===a}function Za(a){return a&&a.$evalAsync&&a.$watch}function Ha(a){return"boolean"===typeof a}function pe(a){return a&&ba(a.length)&&qe.test(ma.call(a))}function Ub(a){return!(!a||!(a.nodeName||a.prop&&a.attr&&a.find))}function re(a){var b={};a=a.split(",");var d;for(d=0;d<a.length;d++)b[a[d]]=!0;return b}function wa(a){return Q(a.nodeName||a[0]&&a[0].nodeName)}function $a(a,b){var d=a.indexOf(b);0<=d&&a.splice(d,1);return d}function ra(a,b,d){function c(a,
-b,c){c--;if(0>c)return"...";var d=b.$$hashKey,f;if(H(a)){f=0;for(var g=a.length;f<g;f++)b.push(e(a[f],c))}else if(Jc(a))for(f in a)b[f]=e(a[f],c);else if(a&&"function"===typeof a.hasOwnProperty)for(f in a)a.hasOwnProperty(f)&&(b[f]=e(a[f],c));else for(f in a)ua.call(a,f)&&(b[f]=e(a[f],c));d?b.$$hashKey=d:delete b.$$hashKey;return b}function e(a,b){if(!C(a))return a;var d=g.indexOf(a);if(-1!==d)return h[d];if(Wa(a)||Za(a))throw Fa("cpws");var d=!1,e=f(a);void 0===e&&(e=H(a)?[]:Object.create(Mc(a)),
-d=!0);g.push(a);h.push(e);return d?c(a,e,b):e}function f(a){switch(ma.call(a)){case "[object Int8Array]":case "[object Int16Array]":case "[object Int32Array]":case "[object Float32Array]":case "[object Float64Array]":case "[object Uint8Array]":case "[object Uint8ClampedArray]":case "[object Uint16Array]":case "[object Uint32Array]":return new a.constructor(e(a.buffer),a.byteOffset,a.length);case "[object ArrayBuffer]":if(!a.slice){var b=new ArrayBuffer(a.byteLength);(new Uint8Array(b)).set(new Uint8Array(a));
-return b}return a.slice(0);case "[object Boolean]":case "[object Number]":case "[object String]":case "[object Date]":return new a.constructor(a.valueOf());case "[object RegExp]":return b=new RegExp(a.source,a.toString().match(/[^/]*$/)[0]),b.lastIndex=a.lastIndex,b;case "[object Blob]":return new a.constructor([a],{type:a.type})}if(D(a.cloneNode))return a.cloneNode(!0)}var g=[],h=[];d=Sb(d)?d:NaN;if(b){if(pe(b)||"[object ArrayBuffer]"===ma.call(b))throw Fa("cpta");if(a===b)throw Fa("cpi");H(b)?b.length=
-0:q(b,function(a,c){"$$hashKey"!==c&&delete b[c]});g.push(a);h.push(b);return c(a,b,d)}return e(a,d)}function Xb(a,b){return a===b||a!==a&&b!==b}function sa(a,b){if(a===b)return!0;if(null===a||null===b)return!1;if(a!==a&&b!==b)return!0;var d=typeof a,c;if(d===typeof b&&"object"===d)if(H(a)){if(!H(b))return!1;if((d=a.length)===b.length){for(c=0;c<d;c++)if(!sa(a[c],b[c]))return!1;return!0}}else{if(ga(a))return ga(b)?Xb(a.getTime(),b.getTime()):!1;if(Xa(a))return Xa(b)?a.toString()===b.toString():!1;
-if(Za(a)||Za(b)||Wa(a)||Wa(b)||H(b)||ga(b)||Xa(b))return!1;d=V();for(c in a)if("$"!==c.charAt(0)&&!D(a[c])){if(!sa(a[c],b[c]))return!1;d[c]=!0}for(c in b)if(!(c in d)&&"$"!==c.charAt(0)&&u(b[c])&&!D(b[c]))return!1;return!0}return!1}function ab(a,b,d){return a.concat(va.call(b,d))}function bb(a,b){var d=2<arguments.length?va.call(arguments,2):[];return!D(b)||b instanceof RegExp?b:d.length?function(){return arguments.length?b.apply(a,ab(d,arguments,0)):b.apply(a,d)}:function(){return arguments.length?
-b.apply(a,arguments):b.call(a)}}function Nc(a,b){var d=b;"string"===typeof a&&"$"===a.charAt(0)&&"$"===a.charAt(1)?d=void 0:Wa(b)?d="$WINDOW":b&&x.document===b?d="$DOCUMENT":Za(b)&&(d="$SCOPE");return d}function cb(a,b){if(!w(a))return ba(b)||(b=b?2:null),JSON.stringify(a,Nc,b)}function Oc(a){return F(a)?JSON.parse(a):a}function Pc(a,b){a=a.replace(se,"");var d=Date.parse("Jan 01, 1970 00:00:00 "+a)/6E4;return da(d)?b:d}function Yb(a,b,d){d=d?-1:1;var c=a.getTimezoneOffset();b=Pc(b,c);d*=b-c;a=new Date(a.getTime());
-a.setMinutes(a.getMinutes()+d);return a}function xa(a){a=B(a).clone();try{a.empty()}catch(b){}var d=B("<div>").append(a).html();try{return a[0].nodeType===Ia?Q(d):d.match(/^(<[^>]+>)/)[1].replace(/^<([\w-]+)/,function(a,b){return"<"+Q(b)})}catch(c){return Q(d)}}function Qc(a){try{return decodeURIComponent(a)}catch(b){}}function Rc(a){var b={};q((a||"").split("&"),function(a){var c,e,f;a&&(e=a=a.replace(/\+/g,"%20"),c=a.indexOf("="),-1!==c&&(e=a.substring(0,c),f=a.substring(c+1)),e=Qc(e),u(e)&&(f=
-u(f)?Qc(f):!0,ua.call(b,e)?H(b[e])?b[e].push(f):b[e]=[b[e],f]:b[e]=f))});return b}function Zb(a){var b=[];q(a,function(a,c){H(a)?q(a,function(a){b.push($(c,!0)+(!0===a?"":"="+$(a,!0)))}):b.push($(c,!0)+(!0===a?"":"="+$(a,!0)))});return b.length?b.join("&"):""}function db(a){return $(a,!0).replace(/%26/gi,"&").replace(/%3D/gi,"=").replace(/%2B/gi,"+")}function $(a,b){return encodeURIComponent(a).replace(/%40/gi,"@").replace(/%3A/gi,":").replace(/%24/g,"$").replace(/%2C/gi,",").replace(/%3B/gi,";").replace(/%20/g,
-b?"%20":"+")}function te(a,b){var d,c,e=Ja.length;for(c=0;c<e;++c)if(d=Ja[c]+b,F(d=a.getAttribute(d)))return d;return null}function ue(a,b){var d,c,e={};q(Ja,function(b){b+="app";!d&&a.hasAttribute&&a.hasAttribute(b)&&(d=a,c=a.getAttribute(b))});q(Ja,function(b){b+="app";var e;!d&&(e=a.querySelector("["+b.replace(":","\\:")+"]"))&&(d=e,c=e.getAttribute(b))});d&&(ve?(e.strictDi=null!==te(d,"strict-di"),b(d,c?[c]:[],e)):x.console.error("Angular: disabling automatic bootstrap. <script> protocol indicates an extension, document.location.href does not match."))}
-function Sc(a,b,d){C(d)||(d={});d=S({strictDi:!1},d);var c=function(){a=B(a);if(a.injector()){var c=a[0]===x.document?"document":xa(a);throw Fa("btstrpd",c.replace(/</,"<").replace(/>/,">"));}b=b||[];b.unshift(["$provide",function(b){b.value("$rootElement",a)}]);d.debugInfoEnabled&&b.push(["$compileProvider",function(a){a.debugInfoEnabled(!0)}]);b.unshift("ng");c=eb(b,d.strictDi);c.invoke(["$rootScope","$rootElement","$compile","$injector",function(a,b,c,d){a.$apply(function(){b.data("$injector",
-d);c(b)(a)})}]);return c},e=/^NG_ENABLE_DEBUG_INFO!/,f=/^NG_DEFER_BOOTSTRAP!/;x&&e.test(x.name)&&(d.debugInfoEnabled=!0,x.name=x.name.replace(e,""));if(x&&!f.test(x.name))return c();x.name=x.name.replace(f,"");ea.resumeBootstrap=function(a){q(a,function(a){b.push(a)});return c()};D(ea.resumeDeferredBootstrap)&&ea.resumeDeferredBootstrap()}function we(){x.name="NG_ENABLE_DEBUG_INFO!"+x.name;x.location.reload()}function xe(a){a=ea.element(a).injector();if(!a)throw Fa("test");return a.get("$$testability")}
-function Tc(a,b){b=b||"_";return a.replace(ye,function(a,c){return(c?b:"")+a.toLowerCase()})}function ze(){var a;if(!Uc){var b=rb();(na=w(b)?x.jQuery:b?x[b]:void 0)&&na.fn.on?(B=na,S(na.fn,{scope:Na.scope,isolateScope:Na.isolateScope,controller:Na.controller,injector:Na.injector,inheritedData:Na.inheritedData}),a=na.cleanData,na.cleanData=function(b){for(var c,e=0,f;null!=(f=b[e]);e++)(c=na._data(f,"events"))&&c.$destroy&&na(f).triggerHandler("$destroy");a(b)}):B=W;ea.element=B;Uc=!0}}function fb(a,
-b,d){if(!a)throw Fa("areq",b||"?",d||"required");return a}function sb(a,b,d){d&&H(a)&&(a=a[a.length-1]);fb(D(a),b,"not a function, got "+(a&&"object"===typeof a?a.constructor.name||"Object":typeof a));return a}function Ka(a,b){if("hasOwnProperty"===a)throw Fa("badname",b);}function Vc(a,b,d){if(!b)return a;b=b.split(".");for(var c,e=a,f=b.length,g=0;g<f;g++)c=b[g],a&&(a=(e=a)[c]);return!d&&D(a)?bb(e,a):a}function tb(a){for(var b=a[0],d=a[a.length-1],c,e=1;b!==d&&(b=b.nextSibling);e++)if(c||a[e]!==
-b)c||(c=B(va.call(a,0,e))),c.push(b);return c||a}function V(){return Object.create(null)}function $b(a){if(null==a)return"";switch(typeof a){case "string":break;case "number":a=""+a;break;default:a=!Wb(a)||H(a)||ga(a)?cb(a):a.toString()}return a}function Ae(a){function b(a,b,c){return a[b]||(a[b]=c())}var d=L("$injector"),c=L("ng");a=b(a,"angular",Object);a.$$minErr=a.$$minErr||L;return b(a,"module",function(){var a={};return function(f,g,h){var k={};if("hasOwnProperty"===f)throw c("badname","module");
-g&&a.hasOwnProperty(f)&&(a[f]=null);return b(a,f,function(){function a(b,c,d,f){f||(f=e);return function(){f[d||"push"]([b,c,arguments]);return v}}function b(a,c,d){d||(d=e);return function(b,e){e&&D(e)&&(e.$$moduleName=f);d.push([a,c,arguments]);return v}}if(!g)throw d("nomod",f);var e=[],p=[],r=[],J=a("$injector","invoke","push",p),v={_invokeQueue:e,_configBlocks:p,_runBlocks:r,info:function(a){if(u(a)){if(!C(a))throw c("aobj","value");k=a;return this}return k},requires:g,name:f,provider:b("$provide",
-"provider"),factory:b("$provide","factory"),service:b("$provide","service"),value:a("$provide","value"),constant:a("$provide","constant","unshift"),decorator:b("$provide","decorator",p),animation:b("$animateProvider","register"),filter:b("$filterProvider","register"),controller:b("$controllerProvider","register"),directive:b("$compileProvider","directive"),component:b("$compileProvider","component"),config:J,run:function(a){r.push(a);return this}};h&&J(h);return v})}})}function pa(a,b){if(H(a)){b=
-b||[];for(var d=0,c=a.length;d<c;d++)b[d]=a[d]}else if(C(a))for(d in b=b||{},a)if("$"!==d.charAt(0)||"$"!==d.charAt(1))b[d]=a[d];return b||a}function Be(a,b){var d=[];Sb(b)&&(a=ra(a,null,b));return JSON.stringify(a,function(a,b){b=Nc(a,b);if(C(b)){if(0<=d.indexOf(b))return"...";d.push(b)}return b})}function Ce(a){S(a,{errorHandlingConfig:me,bootstrap:Sc,copy:ra,extend:S,merge:oe,equals:sa,element:B,forEach:q,injector:eb,noop:z,bind:bb,toJson:cb,fromJson:Oc,identity:Ya,isUndefined:w,isDefined:u,isString:F,
-isFunction:D,isObject:C,isNumber:ba,isElement:Ub,isArray:H,version:De,isDate:ga,lowercase:Q,uppercase:ub,callbacks:{$$counter:0},getTestability:xe,reloadWithDebugInfo:we,$$minErr:L,$$csp:Ga,$$encodeUriSegment:db,$$encodeUriQuery:$,$$stringify:$b});ac=Ae(x);ac("ng",["ngLocale"],["$provide",function(a){a.provider({$$sanitizeUri:Ee});a.provider("$compile",Wc).directive({a:Fe,input:Xc,textarea:Xc,form:Ge,script:He,select:Ie,option:Je,ngBind:Ke,ngBindHtml:Le,ngBindTemplate:Me,ngClass:Ne,ngClassEven:Oe,
-ngClassOdd:Pe,ngCloak:Qe,ngController:Re,ngForm:Se,ngHide:Te,ngIf:Ue,ngInclude:Ve,ngInit:We,ngNonBindable:Xe,ngPluralize:Ye,ngRepeat:Ze,ngShow:$e,ngStyle:af,ngSwitch:bf,ngSwitchWhen:cf,ngSwitchDefault:df,ngOptions:ef,ngTransclude:ff,ngModel:gf,ngList:hf,ngChange:jf,pattern:Yc,ngPattern:Yc,required:Zc,ngRequired:Zc,minlength:$c,ngMinlength:$c,maxlength:ad,ngMaxlength:ad,ngValue:kf,ngModelOptions:lf}).directive({ngInclude:mf}).directive(vb).directive(bd);a.provider({$anchorScroll:nf,$animate:of,$animateCss:pf,
-$$animateJs:qf,$$animateQueue:rf,$$AnimateRunner:sf,$$animateAsyncRun:tf,$browser:uf,$cacheFactory:vf,$controller:wf,$document:xf,$$isDocumentHidden:yf,$exceptionHandler:zf,$filter:cd,$$forceReflow:Af,$interpolate:Bf,$interval:Cf,$http:Df,$httpParamSerializer:Ef,$httpParamSerializerJQLike:Ff,$httpBackend:Gf,$xhrFactory:Hf,$jsonpCallbacks:If,$location:Jf,$log:Kf,$parse:Lf,$rootScope:Mf,$q:Nf,$$q:Of,$sce:Pf,$sceDelegate:Qf,$sniffer:Rf,$templateCache:Sf,$templateRequest:Tf,$$testability:Uf,$timeout:Vf,
-$window:Wf,$$rAF:Xf,$$jqLite:Yf,$$Map:Zf,$$cookieReader:$f})}]).info({angularVersion:"1.6.4"})}function gb(a,b){return b.toUpperCase()}function wb(a){return a.replace(ag,gb)}function bc(a){a=a.nodeType;return 1===a||!a||9===a}function dd(a,b){var d,c,e=b.createDocumentFragment(),f=[];if(cc.test(a)){d=e.appendChild(b.createElement("div"));c=(bg.exec(a)||["",""])[1].toLowerCase();c=ha[c]||ha._default;d.innerHTML=c[1]+a.replace(cg,"<$1></$2>")+c[2];for(c=c[0];c--;)d=d.lastChild;f=ab(f,d.childNodes);
-d=e.firstChild;d.textContent=""}else f.push(b.createTextNode(a));e.textContent="";e.innerHTML="";q(f,function(a){e.appendChild(a)});return e}function W(a){if(a instanceof W)return a;var b;F(a)&&(a=T(a),b=!0);if(!(this instanceof W)){if(b&&"<"!==a.charAt(0))throw dc("nosel");return new W(a)}if(b){b=x.document;var d;a=(d=dg.exec(a))?[b.createElement(d[1])]:(d=dd(a,b))?d.childNodes:[];ec(this,a)}else D(a)?ed(a):ec(this,a)}function fc(a){return a.cloneNode(!0)}function xb(a,b){!b&&bc(a)&&B.cleanData([a]);
-a.querySelectorAll&&B.cleanData(a.querySelectorAll("*"))}function fd(a,b,d,c){if(u(c))throw dc("offargs");var e=(c=yb(a))&&c.events,f=c&&c.handle;if(f)if(b){var g=function(b){var c=e[b];u(d)&&$a(c||[],d);u(d)&&c&&0<c.length||(a.removeEventListener(b,f),delete e[b])};q(b.split(" "),function(a){g(a);zb[a]&&g(zb[a])})}else for(b in e)"$destroy"!==b&&a.removeEventListener(b,f),delete e[b]}function gc(a,b){var d=a.ng339,c=d&&hb[d];c&&(b?delete c.data[b]:(c.handle&&(c.events.$destroy&&c.handle({},"$destroy"),
-fd(a)),delete hb[d],a.ng339=void 0))}function yb(a,b){var d=a.ng339,d=d&&hb[d];b&&!d&&(a.ng339=d=++eg,d=hb[d]={events:{},data:{},handle:void 0});return d}function hc(a,b,d){if(bc(a)){var c,e=u(d),f=!e&&b&&!C(b),g=!b;a=(a=yb(a,!f))&&a.data;if(e)a[wb(b)]=d;else{if(g)return a;if(f)return a&&a[wb(b)];for(c in b)a[wb(c)]=b[c]}}}function Ab(a,b){return a.getAttribute?-1<(" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ").indexOf(" "+b+" "):!1}function Bb(a,b){b&&a.setAttribute&&q(b.split(" "),
-function(b){a.setAttribute("class",T((" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ").replace(" "+T(b)+" "," ")))})}function Cb(a,b){if(b&&a.setAttribute){var d=(" "+(a.getAttribute("class")||"")+" ").replace(/[\n\t]/g," ");q(b.split(" "),function(a){a=T(a);-1===d.indexOf(" "+a+" ")&&(d+=a+" ")});a.setAttribute("class",T(d))}}function ec(a,b){if(b)if(b.nodeType)a[a.length++]=b;else{var d=b.length;if("number"===typeof d&&b.window!==b){if(d)for(var c=0;c<d;c++)a[a.length++]=b[c]}else a[a.length++]=
-b}}function gd(a,b){return Db(a,"$"+(b||"ngController")+"Controller")}function Db(a,b,d){9===a.nodeType&&(a=a.documentElement);for(b=H(b)?b:[b];a;){for(var c=0,e=b.length;c<e;c++)if(u(d=B.data(a,b[c])))return d;a=a.parentNode||11===a.nodeType&&a.host}}function hd(a){for(xb(a,!0);a.firstChild;)a.removeChild(a.firstChild)}function Eb(a,b){b||xb(a);var d=a.parentNode;d&&d.removeChild(a)}function fg(a,b){b=b||x;if("complete"===b.document.readyState)b.setTimeout(a);else B(b).on("load",a)}function ed(a){function b(){x.document.removeEventListener("DOMContentLoaded",
-b);x.removeEventListener("load",b);a()}"complete"===x.document.readyState?x.setTimeout(a):(x.document.addEventListener("DOMContentLoaded",b),x.addEventListener("load",b))}function id(a,b){var d=Fb[b.toLowerCase()];return d&&jd[wa(a)]&&d}function gg(a,b){var d=function(c,d){c.isDefaultPrevented=function(){return c.defaultPrevented};var f=b[d||c.type],g=f?f.length:0;if(g){if(w(c.immediatePropagationStopped)){var h=c.stopImmediatePropagation;c.stopImmediatePropagation=function(){c.immediatePropagationStopped=
-!0;c.stopPropagation&&c.stopPropagation();h&&h.call(c)}}c.isImmediatePropagationStopped=function(){return!0===c.immediatePropagationStopped};var k=f.specialHandlerWrapper||hg;1<g&&(f=pa(f));for(var l=0;l<g;l++)c.isImmediatePropagationStopped()||k(a,c,f[l])}};d.elem=a;return d}function hg(a,b,d){d.call(a,b)}function ig(a,b,d){var c=b.relatedTarget;c&&(c===a||jg.call(a,c))||d.call(a,b)}function Yf(){this.$get=function(){return S(W,{hasClass:function(a,b){a.attr&&(a=a[0]);return Ab(a,b)},addClass:function(a,
-b){a.attr&&(a=a[0]);return Cb(a,b)},removeClass:function(a,b){a.attr&&(a=a[0]);return Bb(a,b)}})}}function Pa(a,b){var d=a&&a.$$hashKey;if(d)return"function"===typeof d&&(d=a.$$hashKey()),d;d=typeof a;return d="function"===d||"object"===d&&null!==a?a.$$hashKey=d+":"+(b||ne)():d+":"+a}function kd(){this._keys=[];this._values=[];this._lastKey=NaN;this._lastIndex=-1}function ld(a){a=Function.prototype.toString.call(a).replace(kg,"");return a.match(lg)||a.match(mg)}function ng(a){return(a=ld(a))?"function("+
-(a[1]||"").replace(/[\s\r\n]+/," ")+")":"fn"}function eb(a,b){function d(a){return function(b,c){if(C(b))q(b,Lc(a));else return a(b,c)}}function c(a,b){Ka(a,"service");if(D(b)||H(b))b=p.instantiate(b);if(!b.$get)throw ya("pget",a);return n[a+"Provider"]=b}function e(a,b){return function(){var c=v.invoke(b,this);if(w(c))throw ya("undef",a);return c}}function f(a,b,d){return c(a,{$get:!1!==d?e(a,b):b})}function g(a){fb(w(a)||H(a),"modulesToLoad","not an array");var b=[],c;q(a,function(a){function d(a){var b,
-c;b=0;for(c=a.length;b<c;b++){var e=a[b],f=p.get(e[0]);f[e[1]].apply(f,e[2])}}if(!m.get(a)){m.set(a,!0);try{F(a)?(c=ac(a),v.modules[a]=c,b=b.concat(g(c.requires)).concat(c._runBlocks),d(c._invokeQueue),d(c._configBlocks)):D(a)?b.push(p.invoke(a)):H(a)?b.push(p.invoke(a)):sb(a,"module")}catch(e){throw H(a)&&(a=a[a.length-1]),e.message&&e.stack&&-1===e.stack.indexOf(e.message)&&(e=e.message+"\n"+e.stack),ya("modulerr",a,e.stack||e.message||e);}}});return b}function h(a,c){function d(b,e){if(a.hasOwnProperty(b)){if(a[b]===
-k)throw ya("cdep",b+" <- "+l.join(" <- "));return a[b]}try{return l.unshift(b),a[b]=k,a[b]=c(b,e),a[b]}catch(f){throw a[b]===k&&delete a[b],f;}finally{l.shift()}}function e(a,c,f){var g=[];a=eb.$$annotate(a,b,f);for(var k=0,h=a.length;k<h;k++){var l=a[k];if("string"!==typeof l)throw ya("itkn",l);g.push(c&&c.hasOwnProperty(l)?c[l]:d(l,f))}return g}return{invoke:function(a,b,c,d){"string"===typeof c&&(d=c,c=null);c=e(a,c,d);H(a)&&(a=a[a.length-1]);d=a;if(za||"function"!==typeof d)d=!1;else{var f=d.$$ngIsClass;
-Ha(f)||(f=d.$$ngIsClass=/^(?:class\b|constructor\()/.test(Function.prototype.toString.call(d)));d=f}return d?(c.unshift(null),new (Function.prototype.bind.apply(a,c))):a.apply(b,c)},instantiate:function(a,b,c){var d=H(a)?a[a.length-1]:a;a=e(a,b,c);a.unshift(null);return new (Function.prototype.bind.apply(d,a))},get:d,annotate:eb.$$annotate,has:function(b){return n.hasOwnProperty(b+"Provider")||a.hasOwnProperty(b)}}}b=!0===b;var k={},l=[],m=new Gb,n={$provide:{provider:d(c),factory:d(f),service:d(function(a,
-b){return f(a,["$injector",function(a){return a.instantiate(b)}])}),value:d(function(a,b){return f(a,la(b),!1)}),constant:d(function(a,b){Ka(a,"constant");n[a]=b;r[a]=b}),decorator:function(a,b){var c=p.get(a+"Provider"),d=c.$get;c.$get=function(){var a=v.invoke(d,c);return v.invoke(b,null,{$delegate:a})}}}},p=n.$injector=h(n,function(a,b){ea.isString(b)&&l.push(b);throw ya("unpr",l.join(" <- "));}),r={},J=h(r,function(a,b){var c=p.get(a+"Provider",b);return v.invoke(c.$get,c,void 0,a)}),v=J;n.$injectorProvider=
-{$get:la(J)};v.modules=p.modules=V();var t=g(a),v=J.get("$injector");v.strictDi=b;q(t,function(a){a&&v.invoke(a)});return v}function nf(){var a=!0;this.disableAutoScrolling=function(){a=!1};this.$get=["$window","$location","$rootScope",function(b,d,c){function e(a){var b=null;Array.prototype.some.call(a,function(a){if("a"===wa(a))return b=a,!0});return b}function f(a){if(a){a.scrollIntoView();var c;c=g.yOffset;D(c)?c=c():Ub(c)?(c=c[0],c="fixed"!==b.getComputedStyle(c).position?0:c.getBoundingClientRect().bottom):
-ba(c)||(c=0);c&&(a=a.getBoundingClientRect().top,b.scrollBy(0,a-c))}else b.scrollTo(0,0)}function g(a){a=F(a)?a:ba(a)?a.toString():d.hash();var b;a?(b=h.getElementById(a))?f(b):(b=e(h.getElementsByName(a)))?f(b):"top"===a&&f(null):f(null)}var h=b.document;a&&c.$watch(function(){return d.hash()},function(a,b){a===b&&""===a||fg(function(){c.$evalAsync(g)})});return g}]}function ib(a,b){if(!a&&!b)return"";if(!a)return b;if(!b)return a;H(a)&&(a=a.join(" "));H(b)&&(b=b.join(" "));return a+" "+b}function og(a){F(a)&&
-(a=a.split(" "));var b=V();q(a,function(a){a.length&&(b[a]=!0)});return b}function ia(a){return C(a)?a:{}}function pg(a,b,d,c){function e(a){try{a.apply(null,va.call(arguments,1))}finally{if(J--,0===J)for(;v.length;)try{v.pop()()}catch(b){d.error(b)}}}function f(){Oa=null;h()}function g(){t=I();t=w(t)?null:t;sa(t,G)&&(t=G);M=G=t}function h(){var a=M;g();if(N!==k.url()||a!==t)N=k.url(),M=t,q(K,function(a){a(k.url(),t)})}var k=this,l=a.location,m=a.history,n=a.setTimeout,p=a.clearTimeout,r={};k.isMock=
-!1;var J=0,v=[];k.$$completeOutstandingRequest=e;k.$$incOutstandingRequestCount=function(){J++};k.notifyWhenNoOutstandingRequests=function(a){0===J?a():v.push(a)};var t,M,N=l.href,A=b.find("base"),Oa=null,I=c.history?function(){try{return m.state}catch(a){}}:z;g();k.url=function(b,d,e){w(e)&&(e=null);l!==a.location&&(l=a.location);m!==a.history&&(m=a.history);if(b){var f=M===e;if(N===b&&(!c.history||f))return k;var h=N&&Aa(N)===Aa(b);N=b;M=e;!c.history||h&&f?(h||(Oa=b),d?l.replace(b):h?(d=l,e=b.indexOf("#"),
-e=-1===e?"":b.substr(e),d.hash=e):l.href=b,l.href!==b&&(Oa=b)):(m[d?"replaceState":"pushState"](e,"",b),g());Oa&&(Oa=b);return k}return Oa||l.href.replace(/%27/g,"'")};k.state=function(){return t};var K=[],E=!1,G=null;k.onUrlChange=function(b){if(!E){if(c.history)B(a).on("popstate",f);B(a).on("hashchange",f);E=!0}K.push(b);return b};k.$$applicationDestroyed=function(){B(a).off("hashchange popstate",f)};k.$$checkUrlChange=h;k.baseHref=function(){var a=A.attr("href");return a?a.replace(/^(https?:)?\/\/[^/]*/,
-""):""};k.defer=function(a,b){var c;J++;c=n(function(){delete r[c];e(a)},b||0);r[c]=!0;return c};k.defer.cancel=function(a){return r[a]?(delete r[a],p(a),e(z),!0):!1}}function uf(){this.$get=["$window","$log","$sniffer","$document",function(a,b,d,c){return new pg(a,c,b,d)}]}function vf(){this.$get=function(){function a(a,c){function e(a){a!==n&&(p?p===a&&(p=a.n):p=a,f(a.n,a.p),f(a,n),n=a,n.n=null)}function f(a,b){a!==b&&(a&&(a.p=b),b&&(b.n=a))}if(a in b)throw L("$cacheFactory")("iid",a);var g=0,h=
-S({},c,{id:a}),k=V(),l=c&&c.capacity||Number.MAX_VALUE,m=V(),n=null,p=null;return b[a]={put:function(a,b){if(!w(b)){if(l<Number.MAX_VALUE){var c=m[a]||(m[a]={key:a});e(c)}a in k||g++;k[a]=b;g>l&&this.remove(p.key);return b}},get:function(a){if(l<Number.MAX_VALUE){var b=m[a];if(!b)return;e(b)}return k[a]},remove:function(a){if(l<Number.MAX_VALUE){var b=m[a];if(!b)return;b===n&&(n=b.p);b===p&&(p=b.n);f(b.n,b.p);delete m[a]}a in k&&(delete k[a],g--)},removeAll:function(){k=V();g=0;m=V();n=p=null},destroy:function(){m=
-h=k=null;delete b[a]},info:function(){return S({},h,{size:g})}}}var b={};a.info=function(){var a={};q(b,function(b,e){a[e]=b.info()});return a};a.get=function(a){return b[a]};return a}}function Sf(){this.$get=["$cacheFactory",function(a){return a("templates")}]}function Wc(a,b){function d(a,b,c){var d=/^\s*([@&<]|=(\*?))(\??)\s*([\w$]*)\s*$/,e=V();q(a,function(a,f){if(a in n)e[f]=n[a];else{var g=a.match(d);if(!g)throw fa("iscp",b,f,a,c?"controller bindings definition":"isolate scope definition");
-e[f]={mode:g[1][0],collection:"*"===g[2],optional:"?"===g[3],attrName:g[4]||f};g[4]&&(n[a]=e[f])}});return e}function c(a){var b=a.charAt(0);if(!b||b!==Q(b))throw fa("baddir",a);if(a!==a.trim())throw fa("baddir",a);}function e(a){var b=a.require||a.controller&&a.name;!H(b)&&C(b)&&q(b,function(a,c){var d=a.match(l);a.substring(d[0].length)||(b[c]=d[0]+c)});return b}var f={},g=/^\s*directive:\s*([\w-]+)\s+(.*)$/,h=/(([\w-]+)(?::([^;]+))?;?)/,k=re("ngSrc,ngSrcset,src,srcset"),l=/^(?:(\^\^?)?(\?)?(\^\^?)?)?/,
-m=/^(on[a-z]+|formaction)$/,n=V();this.directive=function N(b,d){fb(b,"name");Ka(b,"directive");F(b)?(c(b),fb(d,"directiveFactory"),f.hasOwnProperty(b)||(f[b]=[],a.factory(b+"Directive",["$injector","$exceptionHandler",function(a,c){var d=[];q(f[b],function(f,g){try{var h=a.invoke(f);D(h)?h={compile:la(h)}:!h.compile&&h.link&&(h.compile=la(h.link));h.priority=h.priority||0;h.index=g;h.name=h.name||b;h.require=e(h);var k=h,l=h.restrict;if(l&&(!F(l)||!/[EACM]/.test(l)))throw fa("badrestrict",l,b);k.restrict=
-l||"EA";h.$$moduleName=f.$$moduleName;d.push(h)}catch(m){c(m)}});return d}])),f[b].push(d)):q(b,Lc(N));return this};this.component=function(a,b){function c(a){function e(b){return D(b)||H(b)?function(c,d){return a.invoke(b,this,{$element:c,$attrs:d})}:b}var f=b.template||b.templateUrl?b.template:"",g={controller:d,controllerAs:qg(b.controller)||b.controllerAs||"$ctrl",template:e(f),templateUrl:e(b.templateUrl),transclude:b.transclude,scope:{},bindToController:b.bindings||{},restrict:"E",require:b.require};
-q(b,function(a,b){"$"===b.charAt(0)&&(g[b]=a)});return g}var d=b.controller||function(){};q(b,function(a,b){"$"===b.charAt(0)&&(c[b]=a,D(d)&&(d[b]=a))});c.$inject=["$injector"];return this.directive(a,c)};this.aHrefSanitizationWhitelist=function(a){return u(a)?(b.aHrefSanitizationWhitelist(a),this):b.aHrefSanitizationWhitelist()};this.imgSrcSanitizationWhitelist=function(a){return u(a)?(b.imgSrcSanitizationWhitelist(a),this):b.imgSrcSanitizationWhitelist()};var p=!0;this.debugInfoEnabled=function(a){return u(a)?
-(p=a,this):p};var r=!1;this.preAssignBindingsEnabled=function(a){return u(a)?(r=a,this):r};var J=10;this.onChangesTtl=function(a){return arguments.length?(J=a,this):J};var v=!0;this.commentDirectivesEnabled=function(a){return arguments.length?(v=a,this):v};var t=!0;this.cssClassDirectivesEnabled=function(a){return arguments.length?(t=a,this):t};this.$get=["$injector","$interpolate","$exceptionHandler","$templateRequest","$parse","$controller","$rootScope","$sce","$animate","$$sanitizeUri",function(a,
-b,c,e,n,E,G,y,O,X){function P(){try{if(!--ya)throw ia=void 0,fa("infchng",J);G.$apply(function(){for(var a=[],b=0,c=ia.length;b<c;++b)try{ia[b]()}catch(d){a.push(d)}ia=void 0;if(a.length)throw a;})}finally{ya++}}function s(a,b){if(b){var c=Object.keys(b),d,e,f;d=0;for(e=c.length;d<e;d++)f=c[d],this[f]=b[f]}else this.$attr={};this.$$element=a}function R(a,b,c){ta.innerHTML="<span "+b+">";b=ta.firstChild.attributes;var d=b[0];b.removeNamedItem(d.name);d.value=c;a.attributes.setNamedItem(d)}function La(a,
-b){try{a.addClass(b)}catch(c){}}function ca(a,b,c,d,e){a instanceof B||(a=B(a));var f=Ma(a,b,a,c,d,e);ca.$$addScopeClass(a);var g=null;return function(b,c,d){if(!a)throw fa("multilink");fb(b,"scope");e&&e.needsNewScope&&(b=b.$parent.$new());d=d||{};var h=d.parentBoundTranscludeFn,k=d.transcludeControllers;d=d.futureParentElement;h&&h.$$boundTransclude&&(h=h.$$boundTransclude);g||(g=(d=d&&d[0])?"foreignobject"!==wa(d)&&ma.call(d).match(/SVG/)?"svg":"html":"html");d="html"!==g?B(ha(g,B("<div>").append(a).html())):
-c?Na.clone.call(a):a;if(k)for(var l in k)d.data("$"+l+"Controller",k[l].instance);ca.$$addScopeInfo(d,b);c&&c(d,b);f&&f(b,d,d,h);c||(a=f=null);return d}}function Ma(a,b,c,d,e,f){function g(a,c,d,e){var f,k,l,m,n,p,r;if(K)for(r=Array(c.length),m=0;m<h.length;m+=3)f=h[m],r[f]=c[f];else r=c;m=0;for(n=h.length;m<n;)k=r[h[m++]],c=h[m++],f=h[m++],c?(c.scope?(l=a.$new(),ca.$$addScopeInfo(B(k),l)):l=a,p=c.transcludeOnThisElement?ja(a,c.transclude,e):!c.templateOnThisElement&&e?e:!e&&b?ja(a,b):null,c(f,l,
-k,d,p)):f&&f(a,k.childNodes,void 0,e)}for(var h=[],k=H(a)||a instanceof B,l,m,n,p,K,r=0;r<a.length;r++){l=new s;11===za&&L(a,r,k);m=jc(a[r],[],l,0===r?d:void 0,e);(f=m.length?W(m,a[r],l,b,c,null,[],[],f):null)&&f.scope&&ca.$$addScopeClass(l.$$element);l=f&&f.terminal||!(n=a[r].childNodes)||!n.length?null:Ma(n,f?(f.transcludeOnThisElement||!f.templateOnThisElement)&&f.transclude:b);if(f||l)h.push(r,f,l),p=!0,K=K||f;f=null}return p?g:null}function L(a,b,c){var d=a[b],e=d.parentNode,f;if(d.nodeType===
-Ia)for(;;){f=e?d.nextSibling:a[b+1];if(!f||f.nodeType!==Ia)break;d.nodeValue+=f.nodeValue;f.parentNode&&f.parentNode.removeChild(f);c&&f===a[b+1]&&a.splice(b+1,1)}}function ja(a,b,c){function d(e,f,g,h,k){e||(e=a.$new(!1,k),e.$$transcluded=!0);return b(e,f,{parentBoundTranscludeFn:c,transcludeControllers:g,futureParentElement:h})}var e=d.$$slots=V(),f;for(f in b.$$slots)e[f]=b.$$slots[f]?ja(a,b.$$slots[f],c):null;return d}function jc(a,b,c,d,e){var f=c.$attr,g;switch(a.nodeType){case 1:g=wa(a);Y(b,
-Ba(g),"E",d,e);for(var k,l,m,n,p=a.attributes,K=0,r=p&&p.length;K<r;K++){var G=!1,E=!1;k=p[K];l=k.name;m=k.value;k=Ba(l);(n=Ja.test(k))&&(l=l.replace(md,"").substr(8).replace(/_(.)/g,function(a,b){return b.toUpperCase()}));(k=k.match(Ka))&&Z(k[1])&&(G=l,E=l.substr(0,l.length-5)+"end",l=l.substr(0,l.length-6));k=Ba(l.toLowerCase());f[k]=l;if(n||!c.hasOwnProperty(k))c[k]=m,id(a,k)&&(c[k]=!0);pa(a,b,m,k,n);Y(b,k,"A",d,e,G,E)}"input"===g&&"hidden"===a.getAttribute("type")&&a.setAttribute("autocomplete",
-"off");if(!Ga)break;f=a.className;C(f)&&(f=f.animVal);if(F(f)&&""!==f)for(;a=h.exec(f);)k=Ba(a[2]),Y(b,k,"C",d,e)&&(c[k]=T(a[3])),f=f.substr(a.index+a[0].length);break;case Ia:la(b,a.nodeValue);break;case 8:if(!Fa)break;jb(a,b,c,d,e)}b.sort(ea);return b}function jb(a,b,c,d,e){try{var f=g.exec(a.nodeValue);if(f){var h=Ba(f[1]);Y(b,h,"M",d,e)&&(c[h]=T(f[2]))}}catch(k){}}function nd(a,b,c){var d=[],e=0;if(b&&a.hasAttribute&&a.hasAttribute(b)){do{if(!a)throw fa("uterdir",b,c);1===a.nodeType&&(a.hasAttribute(b)&&
-e++,a.hasAttribute(c)&&e--);d.push(a);a=a.nextSibling}while(0<e)}else d.push(a);return B(d)}function od(a,b,c){return function(d,e,f,g,h){e=nd(e[0],b,c);return a(d,e,f,g,h)}}function kc(a,b,c,d,e,f){var g;return a?ca(b,c,d,e,f):function(){g||(g=ca(b,c,d,e,f),b=c=f=null);return g.apply(this,arguments)}}function W(a,b,d,e,f,g,h,k,l){function m(a,b,c,d){if(a){c&&(a=od(a,c,d));a.require=y.require;a.directiveName=P;if(E===y||y.$$isolateScope)a=qa(a,{isolateScope:!0});h.push(a)}if(b){c&&(b=od(b,c,d));b.require=
-y.require;b.directiveName=P;if(E===y||y.$$isolateScope)b=qa(b,{isolateScope:!0});k.push(b)}}function n(a,e,f,g,l){function m(a,b,c,d){var e;Za(a)||(d=c,c=b,b=a,a=void 0);X&&(e=O);c||(c=X?P.parent():P);if(d){var f=l.$$slots[d];if(f)return f(a,b,e,c,R);if(w(f))throw fa("noslot",d,xa(P));}else return l(a,b,e,c,R)}var p,y,t,v,J,O,N,P;b===f?(g=d,P=d.$$element):(P=B(f),g=new s(P,d));J=e;E?v=e.$new(!0):K&&(J=e.$parent);l&&(N=m,N.$$boundTransclude=l,N.isSlotFilled=function(a){return!!l.$$slots[a]});G&&(O=
-ba(P,g,N,G,v,e,E));E&&(ca.$$addScopeInfo(P,v,!0,!(I&&(I===E||I===E.$$originalDirective))),ca.$$addScopeClass(P,!0),v.$$isolateBindings=E.$$isolateBindings,y=na(e,g,v,v.$$isolateBindings,E),y.removeWatches&&v.$on("$destroy",y.removeWatches));for(p in O){y=G[p];t=O[p];var Hb=y.$$bindings.bindToController;if(r){t.bindingInfo=Hb?na(J,g,t.instance,Hb,y):{};var A=t();A!==t.instance&&(t.instance=A,P.data("$"+y.name+"Controller",A),t.bindingInfo.removeWatches&&t.bindingInfo.removeWatches(),t.bindingInfo=
-na(J,g,t.instance,Hb,y))}else t.instance=t(),P.data("$"+y.name+"Controller",t.instance),t.bindingInfo=na(J,g,t.instance,Hb,y)}q(G,function(a,b){var c=a.require;a.bindToController&&!H(c)&&C(c)&&S(O[b].instance,U(b,c,P,O))});q(O,function(a){var b=a.instance;if(D(b.$onChanges))try{b.$onChanges(a.bindingInfo.initialChanges)}catch(d){c(d)}if(D(b.$onInit))try{b.$onInit()}catch(e){c(e)}D(b.$doCheck)&&(J.$watch(function(){b.$doCheck()}),b.$doCheck());D(b.$onDestroy)&&J.$on("$destroy",function(){b.$onDestroy()})});
-p=0;for(y=h.length;p<y;p++)t=h[p],ra(t,t.isolateScope?v:e,P,g,t.require&&U(t.directiveName,t.require,P,O),N);var R=e;E&&(E.template||null===E.templateUrl)&&(R=v);a&&a(R,f.childNodes,void 0,l);for(p=k.length-1;0<=p;p--)t=k[p],ra(t,t.isolateScope?v:e,P,g,t.require&&U(t.directiveName,t.require,P,O),N);q(O,function(a){a=a.instance;D(a.$postLink)&&a.$postLink()})}l=l||{};for(var p=-Number.MAX_VALUE,K=l.newScopeDirective,G=l.controllerDirectives,E=l.newIsolateScopeDirective,I=l.templateDirective,t=l.nonTlbTranscludeDirective,
-J=!1,O=!1,X=l.hasElementTranscludeDirective,v=d.$$element=B(b),y,P,N,A=e,R,u=!1,La=!1,x,z=0,F=a.length;z<F;z++){y=a[z];var Ma=y.$$start,L=y.$$end;Ma&&(v=nd(b,Ma,L));N=void 0;if(p>y.priority)break;if(x=y.scope)y.templateUrl||(C(x)?($("new/isolated scope",E||K,y,v),E=y):$("new/isolated scope",E,y,v)),K=K||y;P=y.name;if(!u&&(y.replace&&(y.templateUrl||y.template)||y.transclude&&!y.$$tlb)){for(x=z+1;u=a[x++];)if(u.transclude&&!u.$$tlb||u.replace&&(u.templateUrl||u.template)){La=!0;break}u=!0}!y.templateUrl&&
-y.controller&&(G=G||V(),$("'"+P+"' controller",G[P],y,v),G[P]=y);if(x=y.transclude)if(J=!0,y.$$tlb||($("transclusion",t,y,v),t=y),"element"===x)X=!0,p=y.priority,N=v,v=d.$$element=B(ca.$$createComment(P,d[P])),b=v[0],ka(f,va.call(N,0),b),N[0].$$parentNode=N[0].parentNode,A=kc(La,N,e,p,g&&g.name,{nonTlbTranscludeDirective:t});else{var ja=V();if(C(x)){N=[];var Q=V(),jb=V();q(x,function(a,b){var c="?"===a.charAt(0);a=c?a.substring(1):a;Q[a]=b;ja[b]=null;jb[b]=c});q(v.contents(),function(a){var b=Q[Ba(wa(a))];
-b?(jb[b]=!0,ja[b]=ja[b]||[],ja[b].push(a)):N.push(a)});q(jb,function(a,b){if(!a)throw fa("reqslot",b);});for(var ic in ja)ja[ic]&&(ja[ic]=kc(La,ja[ic],e))}else N=B(fc(b)).contents();v.empty();A=kc(La,N,e,void 0,void 0,{needsNewScope:y.$$isolateScope||y.$$newScope});A.$$slots=ja}if(y.template)if(O=!0,$("template",I,y,v),I=y,x=D(y.template)?y.template(v,d):y.template,x=Ea(x),y.replace){g=y;N=cc.test(x)?pd(ha(y.templateNamespace,T(x))):[];b=N[0];if(1!==N.length||1!==b.nodeType)throw fa("tplrt",P,"");
-ka(f,v,b);F={$attr:{}};x=jc(b,[],F);var Y=a.splice(z+1,a.length-(z+1));(E||K)&&aa(x,E,K);a=a.concat(x).concat(Y);da(d,F);F=a.length}else v.html(x);if(y.templateUrl)O=!0,$("template",I,y,v),I=y,y.replace&&(g=y),n=ga(a.splice(z,a.length-z),v,d,f,J&&A,h,k,{controllerDirectives:G,newScopeDirective:K!==y&&K,newIsolateScopeDirective:E,templateDirective:I,nonTlbTranscludeDirective:t}),F=a.length;else if(y.compile)try{R=y.compile(v,d,A);var Z=y.$$originalDirective||y;D(R)?m(null,bb(Z,R),Ma,L):R&&m(bb(Z,R.pre),
-bb(Z,R.post),Ma,L)}catch(ea){c(ea,xa(v))}y.terminal&&(n.terminal=!0,p=Math.max(p,y.priority))}n.scope=K&&!0===K.scope;n.transcludeOnThisElement=J;n.templateOnThisElement=O;n.transclude=A;l.hasElementTranscludeDirective=X;return n}function U(a,b,c,d){var e;if(F(b)){var f=b.match(l);b=b.substring(f[0].length);var g=f[1]||f[3],f="?"===f[2];"^^"===g?c=c.parent():e=(e=d&&d[b])&&e.instance;if(!e){var h="$"+b+"Controller";e=g?c.inheritedData(h):c.data(h)}if(!e&&!f)throw fa("ctreq",b,a);}else if(H(b))for(e=
-[],g=0,f=b.length;g<f;g++)e[g]=U(a,b[g],c,d);else C(b)&&(e={},q(b,function(b,f){e[f]=U(a,b,c,d)}));return e||null}function ba(a,b,c,d,e,f,g){var h=V(),k;for(k in d){var l=d[k],m={$scope:l===g||l.$$isolateScope?e:f,$element:a,$attrs:b,$transclude:c},n=l.controller;"@"===n&&(n=b[l.name]);m=E(n,m,!0,l.controllerAs);h[l.name]=m;a.data("$"+l.name+"Controller",m.instance)}return h}function aa(a,b,c){for(var d=0,e=a.length;d<e;d++)a[d]=Vb(a[d],{$$isolateScope:b,$$newScope:c})}function Y(b,c,e,g,h,k,l){if(c===
-h)return null;var m=null;if(f.hasOwnProperty(c)){h=a.get(c+"Directive");for(var n=0,p=h.length;n<p;n++)if(c=h[n],(w(g)||g>c.priority)&&-1!==c.restrict.indexOf(e)){k&&(c=Vb(c,{$$start:k,$$end:l}));if(!c.$$bindings){var K=m=c,r=c.name,t={isolateScope:null,bindToController:null};C(K.scope)&&(!0===K.bindToController?(t.bindToController=d(K.scope,r,!0),t.isolateScope={}):t.isolateScope=d(K.scope,r,!1));C(K.bindToController)&&(t.bindToController=d(K.bindToController,r,!0));if(t.bindToController&&!K.controller)throw fa("noctrl",
-r);m=m.$$bindings=t;C(m.isolateScope)&&(c.$$isolateBindings=m.isolateScope)}b.push(c);m=c}}return m}function Z(b){if(f.hasOwnProperty(b))for(var c=a.get(b+"Directive"),d=0,e=c.length;d<e;d++)if(b=c[d],b.multiElement)return!0;return!1}function da(a,b){var c=b.$attr,d=a.$attr;q(a,function(d,e){"$"!==e.charAt(0)&&(b[e]&&b[e]!==d&&(d=d.length?d+(("style"===e?";":" ")+b[e]):b[e]),a.$set(e,d,!0,c[e]))});q(b,function(b,e){a.hasOwnProperty(e)||"$"===e.charAt(0)||(a[e]=b,"class"!==e&&"style"!==e&&(d[e]=c[e]))})}
-function ga(a,b,d,f,g,h,k,l){var m=[],n,p,K=b[0],r=a.shift(),t=Vb(r,{templateUrl:null,transclude:null,replace:null,$$originalDirective:r}),y=D(r.templateUrl)?r.templateUrl(b,d):r.templateUrl,E=r.templateNamespace;b.empty();e(y).then(function(c){var e,G;c=Ea(c);if(r.replace){c=cc.test(c)?pd(ha(E,T(c))):[];e=c[0];if(1!==c.length||1!==e.nodeType)throw fa("tplrt",r.name,y);c={$attr:{}};ka(f,b,e);var I=jc(e,[],c);C(r.scope)&&aa(I,!0);a=I.concat(a);da(d,c)}else e=K,b.html(c);a.unshift(t);n=W(a,e,d,g,b,
-r,h,k,l);q(f,function(a,c){a===e&&(f[c]=b[0])});for(p=Ma(b[0].childNodes,g);m.length;){c=m.shift();G=m.shift();var v=m.shift(),J=m.shift(),I=b[0];if(!c.$$destroyed){if(G!==K){var O=G.className;l.hasElementTranscludeDirective&&r.replace||(I=fc(e));ka(v,B(G),I);La(B(I),O)}G=n.transcludeOnThisElement?ja(c,n.transclude,J):J;n(p,c,I,f,G)}}m=null}).catch(function(a){a instanceof Error&&c(a)});return function(a,b,c,d,e){a=e;b.$$destroyed||(m?m.push(b,c,d,a):(n.transcludeOnThisElement&&(a=ja(b,n.transclude,
-e)),n(p,b,c,d,a)))}}function ea(a,b){var c=b.priority-a.priority;return 0!==c?c:a.name!==b.name?a.name<b.name?-1:1:a.index-b.index}function $(a,b,c,d){function e(a){return a?" (module: "+a+")":""}if(b)throw fa("multidir",b.name,e(b.$$moduleName),c.name,e(c.$$moduleName),a,xa(d));}function la(a,c){var d=b(c,!0);d&&a.push({priority:0,compile:function(a){a=a.parent();var b=!!a.length;b&&ca.$$addBindingClass(a);return function(a,c){var e=c.parent();b||ca.$$addBindingClass(e);ca.$$addBindingInfo(e,d.expressions);
-a.$watch(d,function(a){c[0].nodeValue=a})}}})}function ha(a,b){a=Q(a||"html");switch(a){case "svg":case "math":var c=x.document.createElement("div");c.innerHTML="<"+a+">"+b+"</"+a+">";return c.childNodes[0].childNodes;default:return b}}function oa(a,b){if("srcdoc"===b)return y.HTML;var c=wa(a);if("src"===b||"ngSrc"===b){if(-1===["img","video","audio","source","track"].indexOf(c))return y.RESOURCE_URL}else if("xlinkHref"===b||"form"===c&&"action"===b||"link"===c&&"href"===b)return y.RESOURCE_URL}function pa(a,
-c,d,e,f){var g=oa(a,e),h=k[e]||f,l=b(d,!f,g,h);if(l){if("multiple"===e&&"select"===wa(a))throw fa("selmulti",xa(a));if(m.test(e))throw fa("nodomevents");c.push({priority:100,compile:function(){return{pre:function(a,c,f){c=f.$$observers||(f.$$observers=V());var k=f[e];k!==d&&(l=k&&b(k,!0,g,h),d=k);l&&(f[e]=l(a),(c[e]||(c[e]=[])).$$inter=!0,(f.$$observers&&f.$$observers[e].$$scope||a).$watch(l,function(a,b){"class"===e&&a!==b?f.$updateClass(a,b):f.$set(e,a)}))}}}})}}function ka(a,b,c){var d=b[0],e=
-b.length,f=d.parentNode,g,h;if(a)for(g=0,h=a.length;g<h;g++)if(a[g]===d){a[g++]=c;h=g+e-1;for(var k=a.length;g<k;g++,h++)h<k?a[g]=a[h]:delete a[g];a.length-=e-1;a.context===d&&(a.context=c);break}f&&f.replaceChild(c,d);a=x.document.createDocumentFragment();for(g=0;g<e;g++)a.appendChild(b[g]);B.hasData(d)&&(B.data(c,B.data(d)),B(d).off("$destroy"));B.cleanData(a.querySelectorAll("*"));for(g=1;g<e;g++)delete b[g];b[0]=c;b.length=1}function qa(a,b){return S(function(){return a.apply(null,arguments)},
-a,b)}function ra(a,b,d,e,f,g){try{a(b,d,e,f,g)}catch(h){c(h,xa(d))}}function na(a,c,d,e,f){function g(b,c,e){D(d.$onChanges)&&!Xb(c,e)&&(ia||(a.$$postDigest(P),ia=[]),m||(m={},ia.push(h)),m[b]&&(e=m[b].previousValue),m[b]=new Ib(e,c))}function h(){d.$onChanges(m);m=void 0}var k=[],l={},m;q(e,function(e,h){var m=e.attrName,p=e.optional,r,t,y,G;switch(e.mode){case "@":p||ua.call(c,m)||(d[h]=c[m]=void 0);p=c.$observe(m,function(a){if(F(a)||Ha(a))g(h,a,d[h]),d[h]=a});c.$$observers[m].$$scope=a;r=c[m];
-F(r)?d[h]=b(r)(a):Ha(r)&&(d[h]=r);l[h]=new Ib(lc,d[h]);k.push(p);break;case "=":if(!ua.call(c,m)){if(p)break;c[m]=void 0}if(p&&!c[m])break;t=n(c[m]);G=t.literal?sa:Xb;y=t.assign||function(){r=d[h]=t(a);throw fa("nonassign",c[m],m,f.name);};r=d[h]=t(a);p=function(b){G(b,d[h])||(G(b,r)?y(a,b=d[h]):d[h]=b);return r=b};p.$stateful=!0;p=e.collection?a.$watchCollection(c[m],p):a.$watch(n(c[m],p),null,t.literal);k.push(p);break;case "<":if(!ua.call(c,m)){if(p)break;c[m]=void 0}if(p&&!c[m])break;t=n(c[m]);
-var E=t.literal,I=d[h]=t(a);l[h]=new Ib(lc,d[h]);p=a.$watch(t,function(a,b){if(b===a){if(b===I||E&&sa(b,I))return;b=I}g(h,a,b);d[h]=a},E);k.push(p);break;case "&":t=c.hasOwnProperty(m)?n(c[m]):z;if(t===z&&p)break;d[h]=function(b){return t(a,b)}}});return{initialChanges:l,removeWatches:k.length&&function(){for(var a=0,b=k.length;a<b;++a)k[a]()}}}var Ca=/^\w/,ta=x.document.createElement("div"),Fa=v,Ga=t,ya=J,ia;s.prototype={$normalize:Ba,$addClass:function(a){a&&0<a.length&&O.addClass(this.$$element,
-a)},$removeClass:function(a){a&&0<a.length&&O.removeClass(this.$$element,a)},$updateClass:function(a,b){var c=qd(a,b);c&&c.length&&O.addClass(this.$$element,c);(c=qd(b,a))&&c.length&&O.removeClass(this.$$element,c)},$set:function(a,b,d,e){var f=id(this.$$element[0],a),g=rd[a],h=a;f?(this.$$element.prop(a,b),e=f):g&&(this[g]=b,h=g);this[a]=b;e?this.$attr[a]=e:(e=this.$attr[a])||(this.$attr[a]=e=Tc(a,"-"));f=wa(this.$$element);if("a"===f&&("href"===a||"xlinkHref"===a)||"img"===f&&"src"===a)this[a]=
-b=X(b,"src"===a);else if("img"===f&&"srcset"===a&&u(b)){for(var f="",g=T(b),k=/(\s+\d+x\s*,|\s+\d+w\s*,|\s+,|,\s+)/,k=/\s/.test(g)?k:/(,)/,g=g.split(k),k=Math.floor(g.length/2),l=0;l<k;l++)var m=2*l,f=f+X(T(g[m]),!0),f=f+(" "+T(g[m+1]));g=T(g[2*l]).split(/\s/);f+=X(T(g[0]),!0);2===g.length&&(f+=" "+T(g[1]));this[a]=b=f}!1!==d&&(null===b||w(b)?this.$$element.removeAttr(e):Ca.test(e)?this.$$element.attr(e,b):R(this.$$element[0],e,b));(a=this.$$observers)&&q(a[h],function(a){try{a(b)}catch(d){c(d)}})},
-$observe:function(a,b){var c=this,d=c.$$observers||(c.$$observers=V()),e=d[a]||(d[a]=[]);e.push(b);G.$evalAsync(function(){e.$$inter||!c.hasOwnProperty(a)||w(c[a])||b(c[a])});return function(){$a(e,b)}}};var Aa=b.startSymbol(),Da=b.endSymbol(),Ea="{{"===Aa&&"}}"===Da?Ya:function(a){return a.replace(/\{\{/g,Aa).replace(/}}/g,Da)},Ja=/^ngAttr[A-Z]/,Ka=/^(.+)Start$/;ca.$$addBindingInfo=p?function(a,b){var c=a.data("$binding")||[];H(b)?c=c.concat(b):c.push(b);a.data("$binding",c)}:z;ca.$$addBindingClass=
-p?function(a){La(a,"ng-binding")}:z;ca.$$addScopeInfo=p?function(a,b,c,d){a.data(c?d?"$isolateScopeNoTemplate":"$isolateScope":"$scope",b)}:z;ca.$$addScopeClass=p?function(a,b){La(a,b?"ng-isolate-scope":"ng-scope")}:z;ca.$$createComment=function(a,b){var c="";p&&(c=" "+(a||"")+": ",b&&(c+=b+" "));return x.document.createComment(c)};return ca}]}function Ib(a,b){this.previousValue=a;this.currentValue=b}function Ba(a){return a.replace(md,"").replace(rg,gb)}function qd(a,b){var d="",c=a.split(/\s+/),
-e=b.split(/\s+/),f=0;a:for(;f<c.length;f++){for(var g=c[f],h=0;h<e.length;h++)if(g===e[h])continue a;d+=(0<d.length?" ":"")+g}return d}function pd(a){a=B(a);var b=a.length;if(1>=b)return a;for(;b--;){var d=a[b];(8===d.nodeType||d.nodeType===Ia&&""===d.nodeValue.trim())&&sg.call(a,b,1)}return a}function qg(a,b){if(b&&F(b))return b;if(F(a)){var d=sd.exec(a);if(d)return d[3]}}function wf(){var a={},b=!1;this.has=function(b){return a.hasOwnProperty(b)};this.register=function(b,c){Ka(b,"controller");C(b)?
-S(a,b):a[b]=c};this.allowGlobals=function(){b=!0};this.$get=["$injector","$window",function(d,c){function e(a,b,c,d){if(!a||!C(a.$scope))throw L("$controller")("noscp",d,b);a.$scope[b]=c}return function(f,g,h,k){var l,m,n;h=!0===h;k&&F(k)&&(n=k);if(F(f)){k=f.match(sd);if(!k)throw td("ctrlfmt",f);m=k[1];n=n||k[3];f=a.hasOwnProperty(m)?a[m]:Vc(g.$scope,m,!0)||(b?Vc(c,m,!0):void 0);if(!f)throw td("ctrlreg",m);sb(f,m,!0)}if(h)return h=(H(f)?f[f.length-1]:f).prototype,l=Object.create(h||null),n&&e(g,n,
-l,m||f.name),S(function(){var a=d.invoke(f,l,g,m);a!==l&&(C(a)||D(a))&&(l=a,n&&e(g,n,l,m||f.name));return l},{instance:l,identifier:n});l=d.instantiate(f,g,m);n&&e(g,n,l,m||f.name);return l}}]}function xf(){this.$get=["$window",function(a){return B(a.document)}]}function yf(){this.$get=["$document","$rootScope",function(a,b){function d(){e=c.hidden}var c=a[0],e=c&&c.hidden;a.on("visibilitychange",d);b.$on("$destroy",function(){a.off("visibilitychange",d)});return function(){return e}}]}function zf(){this.$get=
-["$log",function(a){return function(b,d){a.error.apply(a,arguments)}}]}function mc(a){return C(a)?ga(a)?a.toISOString():cb(a):a}function Ef(){this.$get=function(){return function(a){if(!a)return"";var b=[];Kc(a,function(a,c){null===a||w(a)||(H(a)?q(a,function(a){b.push($(c)+"="+$(mc(a)))}):b.push($(c)+"="+$(mc(a))))});return b.join("&")}}}function Ff(){this.$get=function(){return function(a){function b(a,e,f){null===a||w(a)||(H(a)?q(a,function(a,c){b(a,e+"["+(C(a)?c:"")+"]")}):C(a)&&!ga(a)?Kc(a,function(a,
-c){b(a,e+(f?"":"[")+c+(f?"":"]"))}):d.push($(e)+"="+$(mc(a))))}if(!a)return"";var d=[];b(a,"",!0);return d.join("&")}}}function nc(a,b){if(F(a)){var d=a.replace(tg,"").trim();if(d){var c=b("Content-Type");(c=c&&0===c.indexOf(ud))||(c=(c=d.match(ug))&&vg[c[0]].test(d));if(c)try{a=Oc(d)}catch(e){throw oc("baddata",a,e);}}}return a}function vd(a){var b=V(),d;F(a)?q(a.split("\n"),function(a){d=a.indexOf(":");var e=Q(T(a.substr(0,d)));a=T(a.substr(d+1));e&&(b[e]=b[e]?b[e]+", "+a:a)}):C(a)&&q(a,function(a,
-d){var f=Q(d),g=T(a);f&&(b[f]=b[f]?b[f]+", "+g:g)});return b}function wd(a){var b;return function(d){b||(b=vd(a));return d?(d=b[Q(d)],void 0===d&&(d=null),d):b}}function xd(a,b,d,c){if(D(c))return c(a,b,d);q(c,function(c){a=c(a,b,d)});return a}function Df(){var a=this.defaults={transformResponse:[nc],transformRequest:[function(a){return C(a)&&"[object File]"!==ma.call(a)&&"[object Blob]"!==ma.call(a)&&"[object FormData]"!==ma.call(a)?cb(a):a}],headers:{common:{Accept:"application/json, text/plain, */*"},
-post:pa(pc),put:pa(pc),patch:pa(pc)},xsrfCookieName:"XSRF-TOKEN",xsrfHeaderName:"X-XSRF-TOKEN",paramSerializer:"$httpParamSerializer",jsonpCallbackParam:"callback"},b=!1;this.useApplyAsync=function(a){return u(a)?(b=!!a,this):b};var d=this.interceptors=[];this.$get=["$browser","$httpBackend","$$cookieReader","$cacheFactory","$rootScope","$q","$injector","$sce",function(c,e,f,g,h,k,l,m){function n(b){function d(a,b){for(var c=0,e=b.length;c<e;){var f=b[c++],g=b[c++];a=a.then(f,g)}b.length=0;return a}
-function e(a,b){var c,d={};q(a,function(a,e){D(a)?(c=a(b),null!=c&&(d[e]=c)):d[e]=a});return d}function f(a){var b=S({},a);b.data=xd(a.data,a.headers,a.status,g.transformResponse);a=a.status;return 200<=a&&300>a?b:k.reject(b)}if(!C(b))throw L("$http")("badreq",b);if(!F(m.valueOf(b.url)))throw L("$http")("badreq",b.url);var g=S({method:"get",transformRequest:a.transformRequest,transformResponse:a.transformResponse,paramSerializer:a.paramSerializer,jsonpCallbackParam:a.jsonpCallbackParam},b);g.headers=
-function(b){var c=a.headers,d=S({},b.headers),f,g,h,c=S({},c.common,c[Q(b.method)]);a:for(f in c){g=Q(f);for(h in d)if(Q(h)===g)continue a;d[f]=c[f]}return e(d,pa(b))}(b);g.method=ub(g.method);g.paramSerializer=F(g.paramSerializer)?l.get(g.paramSerializer):g.paramSerializer;c.$$incOutstandingRequestCount();var h=[],n=[];b=k.resolve(g);q(t,function(a){(a.request||a.requestError)&&h.unshift(a.request,a.requestError);(a.response||a.responseError)&&n.push(a.response,a.responseError)});b=d(b,h);b=b.then(function(b){var c=
-b.headers,d=xd(b.data,wd(c),void 0,b.transformRequest);w(d)&&q(c,function(a,b){"content-type"===Q(b)&&delete c[b]});w(b.withCredentials)&&!w(a.withCredentials)&&(b.withCredentials=a.withCredentials);return p(b,d).then(f,f)});b=d(b,n);return b=b.finally(function(){c.$$completeOutstandingRequest(z)})}function p(c,d){function g(a){if(a){var c={};q(a,function(a,d){c[d]=function(c){function d(){a(c)}b?h.$applyAsync(d):h.$$phase?d():h.$apply(d)}});return c}}function l(a,c,d,e){function f(){p(c,a,d,e)}O&&
-(200<=a&&300>a?O.put(R,[a,c,vd(d),e]):O.remove(R));b?h.$applyAsync(f):(f(),h.$$phase||h.$apply())}function p(a,b,d,e){b=-1<=b?b:0;(200<=b&&300>b?G.resolve:G.reject)({data:a,status:b,headers:wd(d),config:c,statusText:e})}function K(a){p(a.data,a.status,pa(a.headers()),a.statusText)}function t(){var a=n.pendingRequests.indexOf(c);-1!==a&&n.pendingRequests.splice(a,1)}var G=k.defer(),y=G.promise,O,X,P=c.headers,s="jsonp"===Q(c.method),R=c.url;s?R=m.getTrustedResourceUrl(R):F(R)||(R=m.valueOf(R));R=r(R,
-c.paramSerializer(c.params));s&&(R=J(R,c.jsonpCallbackParam));n.pendingRequests.push(c);y.then(t,t);!c.cache&&!a.cache||!1===c.cache||"GET"!==c.method&&"JSONP"!==c.method||(O=C(c.cache)?c.cache:C(a.cache)?a.cache:v);O&&(X=O.get(R),u(X)?X&&D(X.then)?X.then(K,K):H(X)?p(X[1],X[0],pa(X[2]),X[3]):p(X,200,{},"OK"):O.put(R,y));w(X)&&((X=yd(c.url)?f()[c.xsrfCookieName||a.xsrfCookieName]:void 0)&&(P[c.xsrfHeaderName||a.xsrfHeaderName]=X),e(c.method,R,d,l,P,c.timeout,c.withCredentials,c.responseType,g(c.eventHandlers),
-g(c.uploadEventHandlers)));return y}function r(a,b){0<b.length&&(a+=(-1===a.indexOf("?")?"?":"&")+b);return a}function J(a,b){if(/[&?][^=]+=JSON_CALLBACK/.test(a))throw oc("badjsonp",a);if((new RegExp("[&?]"+b+"=")).test(a))throw oc("badjsonp",b,a);return a+=(-1===a.indexOf("?")?"?":"&")+b+"=JSON_CALLBACK"}var v=g("$http");a.paramSerializer=F(a.paramSerializer)?l.get(a.paramSerializer):a.paramSerializer;var t=[];q(d,function(a){t.unshift(F(a)?l.get(a):l.invoke(a))});n.pendingRequests=[];(function(a){q(arguments,
-function(a){n[a]=function(b,c){return n(S({},c||{},{method:a,url:b}))}})})("get","delete","head","jsonp");(function(a){q(arguments,function(a){n[a]=function(b,c,d){return n(S({},d||{},{method:a,url:b,data:c}))}})})("post","put","patch");n.defaults=a;return n}]}function Hf(){this.$get=function(){return function(){return new x.XMLHttpRequest}}}function Gf(){this.$get=["$browser","$jsonpCallbacks","$document","$xhrFactory",function(a,b,d,c){return wg(a,c,a.defer,b,d[0])}]}function wg(a,b,d,c,e){function f(a,
-b,d){a=a.replace("JSON_CALLBACK",b);var f=e.createElement("script"),m=null;f.type="text/javascript";f.src=a;f.async=!0;m=function(a){f.removeEventListener("load",m);f.removeEventListener("error",m);e.body.removeChild(f);f=null;var g=-1,r="unknown";a&&("load"!==a.type||c.wasCalled(b)||(a={type:"error"}),r=a.type,g="error"===a.type?404:200);d&&d(g,r)};f.addEventListener("load",m);f.addEventListener("error",m);e.body.appendChild(f);return m}return function(e,h,k,l,m,n,p,r,J,v){function t(){N&&N();A&&
-A.abort()}h=h||a.url();if("jsonp"===Q(e))var M=c.createCallback(h),N=f(h,M,function(a,b){var e=200===a&&c.getResponse(M);u(I)&&d.cancel(I);N=A=null;l(a,e,"",b);c.removeCallback(M)});else{var A=b(e,h);A.open(e,h,!0);q(m,function(a,b){u(a)&&A.setRequestHeader(b,a)});A.onload=function(){var a=A.statusText||"",b="response"in A?A.response:A.responseText,c=1223===A.status?204:A.status;0===c&&(c=b?200:"file"===Ca(h).protocol?404:0);var e=A.getAllResponseHeaders();u(I)&&d.cancel(I);N=A=null;l(c,b,e,a)};e=
-function(){u(I)&&d.cancel(I);N=A=null;l(-1,null,null,"")};A.onerror=e;A.onabort=e;A.ontimeout=e;q(J,function(a,b){A.addEventListener(b,a)});q(v,function(a,b){A.upload.addEventListener(b,a)});p&&(A.withCredentials=!0);if(r)try{A.responseType=r}catch(s){if("json"!==r)throw s;}A.send(w(k)?null:k)}if(0<n)var I=d(t,n);else n&&D(n.then)&&n.then(t)}}function Bf(){var a="{{",b="}}";this.startSymbol=function(b){return b?(a=b,this):a};this.endSymbol=function(a){return a?(b=a,this):b};this.$get=["$parse","$exceptionHandler",
-"$sce",function(d,c,e){function f(a){return"\\\\\\"+a}function g(c){return c.replace(n,a).replace(p,b)}function h(a,b,c,d){var e=a.$watch(function(a){e();return d(a)},b,c);return e}function k(f,k,n,p){function M(a){try{var b=a;a=n?e.getTrusted(n,b):e.valueOf(b);return p&&!u(a)?a:$b(a)}catch(d){c(Da.interr(f,d))}}if(!f.length||-1===f.indexOf(a)){var q;k||(k=g(f),q=la(k),q.exp=f,q.expressions=[],q.$$watchDelegate=h);return q}p=!!p;var A,s,I=0,K=[],E=[];q=f.length;for(var G=[],y=[];I<q;)if(-1!==(A=f.indexOf(a,
-I))&&-1!==(s=f.indexOf(b,A+l)))I!==A&&G.push(g(f.substring(I,A))),I=f.substring(A+l,s),K.push(I),E.push(d(I,M)),I=s+m,y.push(G.length),G.push("");else{I!==q&&G.push(g(f.substring(I)));break}n&&1<G.length&&Da.throwNoconcat(f);if(!k||K.length){var O=function(a){for(var b=0,c=K.length;b<c;b++){if(p&&w(a[b]))return;G[y[b]]=a[b]}return G.join("")};return S(function(a){var b=0,d=K.length,e=Array(d);try{for(;b<d;b++)e[b]=E[b](a);return O(e)}catch(g){c(Da.interr(f,g))}},{exp:f,expressions:K,$$watchDelegate:function(a,
-b){var c;return a.$watchGroup(E,function(d,e){var f=O(d);D(b)&&b.call(this,f,d!==e?c:f,a);c=f})}})}}var l=a.length,m=b.length,n=new RegExp(a.replace(/./g,f),"g"),p=new RegExp(b.replace(/./g,f),"g");k.startSymbol=function(){return a};k.endSymbol=function(){return b};return k}]}function Cf(){this.$get=["$rootScope","$window","$q","$$q","$browser",function(a,b,d,c,e){function f(f,k,l,m){function n(){p?f.apply(null,r):f(t)}var p=4<arguments.length,r=p?va.call(arguments,4):[],J=b.setInterval,v=b.clearInterval,
-t=0,M=u(m)&&!m,q=(M?c:d).defer(),A=q.promise;l=u(l)?l:0;A.$$intervalId=J(function(){M?e.defer(n):a.$evalAsync(n);q.notify(t++);0<l&&t>=l&&(q.resolve(t),v(A.$$intervalId),delete g[A.$$intervalId]);M||a.$apply()},k);g[A.$$intervalId]=q;return A}var g={};f.cancel=function(a){return a&&a.$$intervalId in g?(g[a.$$intervalId].promise.catch(z),g[a.$$intervalId].reject("canceled"),b.clearInterval(a.$$intervalId),delete g[a.$$intervalId],!0):!1};return f}]}function qc(a){a=a.split("/");for(var b=a.length;b--;)a[b]=
-db(a[b]);return a.join("/")}function zd(a,b){var d=Ca(a);b.$$protocol=d.protocol;b.$$host=d.hostname;b.$$port=Z(d.port)||xg[d.protocol]||null}function Ad(a,b){if(yg.test(a))throw kb("badpath",a);var d="/"!==a.charAt(0);d&&(a="/"+a);var c=Ca(a);b.$$path=decodeURIComponent(d&&"/"===c.pathname.charAt(0)?c.pathname.substring(1):c.pathname);b.$$search=Rc(c.search);b.$$hash=decodeURIComponent(c.hash);b.$$path&&"/"!==b.$$path.charAt(0)&&(b.$$path="/"+b.$$path)}function rc(a,b){return a.slice(0,b.length)===
-b}function ka(a,b){if(rc(b,a))return b.substr(a.length)}function Aa(a){var b=a.indexOf("#");return-1===b?a:a.substr(0,b)}function lb(a){return a.replace(/(#.+)|#$/,"$1")}function sc(a,b,d){this.$$html5=!0;d=d||"";zd(a,this);this.$$parse=function(a){var d=ka(b,a);if(!F(d))throw kb("ipthprfx",a,b);Ad(d,this);this.$$path||(this.$$path="/");this.$$compose()};this.$$compose=function(){var a=Zb(this.$$search),d=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(a?"?"+a:"")+d;this.$$absUrl=b+
-this.$$url.substr(1);this.$$urlUpdatedByLocation=!0};this.$$parseLinkUrl=function(c,e){if(e&&"#"===e[0])return this.hash(e.slice(1)),!0;var f,g;u(f=ka(a,c))?(g=f,g=d&&u(f=ka(d,f))?b+(ka("/",f)||f):a+g):u(f=ka(b,c))?g=b+f:b===c+"/"&&(g=b);g&&this.$$parse(g);return!!g}}function tc(a,b,d){zd(a,this);this.$$parse=function(c){var e=ka(a,c)||ka(b,c),f;w(e)||"#"!==e.charAt(0)?this.$$html5?f=e:(f="",w(e)&&(a=c,this.replace())):(f=ka(d,e),w(f)&&(f=e));Ad(f,this);c=this.$$path;var e=a,g=/^\/[A-Z]:(\/.*)/;rc(f,
-e)&&(f=f.replace(e,""));g.exec(f)||(c=(f=g.exec(c))?f[1]:c);this.$$path=c;this.$$compose()};this.$$compose=function(){var b=Zb(this.$$search),e=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(b?"?"+b:"")+e;this.$$absUrl=a+(this.$$url?d+this.$$url:"");this.$$urlUpdatedByLocation=!0};this.$$parseLinkUrl=function(b,d){return Aa(a)===Aa(b)?(this.$$parse(b),!0):!1}}function Bd(a,b,d){this.$$html5=!0;tc.apply(this,arguments);this.$$parseLinkUrl=function(c,e){if(e&&"#"===e[0])return this.hash(e.slice(1)),
-!0;var f,g;a===Aa(c)?f=c:(g=ka(b,c))?f=a+d+g:b===c+"/"&&(f=b);f&&this.$$parse(f);return!!f};this.$$compose=function(){var b=Zb(this.$$search),e=this.$$hash?"#"+db(this.$$hash):"";this.$$url=qc(this.$$path)+(b?"?"+b:"")+e;this.$$absUrl=a+d+this.$$url;this.$$urlUpdatedByLocation=!0}}function Jb(a){return function(){return this[a]}}function Cd(a,b){return function(d){if(w(d))return this[a];this[a]=b(d);this.$$compose();return this}}function Jf(){var a="!",b={enabled:!1,requireBase:!0,rewriteLinks:!0};
-this.hashPrefix=function(b){return u(b)?(a=b,this):a};this.html5Mode=function(a){if(Ha(a))return b.enabled=a,this;if(C(a)){Ha(a.enabled)&&(b.enabled=a.enabled);Ha(a.requireBase)&&(b.requireBase=a.requireBase);if(Ha(a.rewriteLinks)||F(a.rewriteLinks))b.rewriteLinks=a.rewriteLinks;return this}return b};this.$get=["$rootScope","$browser","$sniffer","$rootElement","$window",function(d,c,e,f,g){function h(a,b,d){var e=l.url(),f=l.$$state;try{c.url(a,b,d),l.$$state=c.state()}catch(g){throw l.url(e),l.$$state=
-f,g;}}function k(a,b){d.$broadcast("$locationChangeSuccess",l.absUrl(),a,l.$$state,b)}var l,m;m=c.baseHref();var n=c.url(),p;if(b.enabled){if(!m&&b.requireBase)throw kb("nobase");p=n.substring(0,n.indexOf("/",n.indexOf("//")+2))+(m||"/");m=e.history?sc:Bd}else p=Aa(n),m=tc;var r=p.substr(0,Aa(p).lastIndexOf("/")+1);l=new m(p,r,"#"+a);l.$$parseLinkUrl(n,n);l.$$state=c.state();var J=/^\s*(javascript|mailto):/i;f.on("click",function(a){var e=b.rewriteLinks;if(e&&!a.ctrlKey&&!a.metaKey&&!a.shiftKey&&
-2!==a.which&&2!==a.button){for(var h=B(a.target);"a"!==wa(h[0]);)if(h[0]===f[0]||!(h=h.parent())[0])return;if(!F(e)||!w(h.attr(e))){var e=h.prop("href"),k=h.attr("href")||h.attr("xlink:href");C(e)&&"[object SVGAnimatedString]"===e.toString()&&(e=Ca(e.animVal).href);J.test(e)||!e||h.attr("target")||a.isDefaultPrevented()||!l.$$parseLinkUrl(e,k)||(a.preventDefault(),l.absUrl()!==c.url()&&(d.$apply(),g.angular["ff-684208-preventDefault"]=!0))}}});lb(l.absUrl())!==lb(n)&&c.url(l.absUrl(),!0);var v=!0;
-c.onUrlChange(function(a,b){rc(a,r)?(d.$evalAsync(function(){var c=l.absUrl(),e=l.$$state,f;a=lb(a);l.$$parse(a);l.$$state=b;f=d.$broadcast("$locationChangeStart",a,c,b,e).defaultPrevented;l.absUrl()===a&&(f?(l.$$parse(c),l.$$state=e,h(c,!1,e)):(v=!1,k(c,e)))}),d.$$phase||d.$digest()):g.location.href=a});d.$watch(function(){if(v||l.$$urlUpdatedByLocation){l.$$urlUpdatedByLocation=!1;var a=lb(c.url()),b=lb(l.absUrl()),f=c.state(),g=l.$$replace,m=a!==b||l.$$html5&&e.history&&f!==l.$$state;if(v||m)v=
-!1,d.$evalAsync(function(){var b=l.absUrl(),c=d.$broadcast("$locationChangeStart",b,a,l.$$state,f).defaultPrevented;l.absUrl()===b&&(c?(l.$$parse(a),l.$$state=f):(m&&h(b,g,f===l.$$state?null:l.$$state),k(a,f)))})}l.$$replace=!1});return l}]}function Kf(){var a=!0,b=this;this.debugEnabled=function(b){return u(b)?(a=b,this):a};this.$get=["$window",function(d){function c(a){a instanceof Error&&(a.stack&&f?a=a.message&&-1===a.stack.indexOf(a.message)?"Error: "+a.message+"\n"+a.stack:a.stack:a.sourceURL&&
-(a=a.message+"\n"+a.sourceURL+":"+a.line));return a}function e(a){var b=d.console||{},e=b[a]||b.log||z;a=!1;try{a=!!e.apply}catch(f){}return a?function(){var a=[];q(arguments,function(b){a.push(c(b))});return e.apply(b,a)}:function(a,b){e(a,null==b?"":b)}}var f=za||/\bEdge\//.test(d.navigator&&d.navigator.userAgent);return{log:e("log"),info:e("info"),warn:e("warn"),error:e("error"),debug:function(){var c=e("debug");return function(){a&&c.apply(b,arguments)}}()}}]}function zg(a){return a+""}function Ag(a,
-b){return"undefined"!==typeof a?a:b}function Dd(a,b){return"undefined"===typeof a?b:"undefined"===typeof b?a:a+b}function U(a,b){var d,c,e;switch(a.type){case s.Program:d=!0;q(a.body,function(a){U(a.expression,b);d=d&&a.expression.constant});a.constant=d;break;case s.Literal:a.constant=!0;a.toWatch=[];break;case s.UnaryExpression:U(a.argument,b);a.constant=a.argument.constant;a.toWatch=a.argument.toWatch;break;case s.BinaryExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;
-a.toWatch=a.left.toWatch.concat(a.right.toWatch);break;case s.LogicalExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;a.toWatch=a.constant?[]:[a];break;case s.ConditionalExpression:U(a.test,b);U(a.alternate,b);U(a.consequent,b);a.constant=a.test.constant&&a.alternate.constant&&a.consequent.constant;a.toWatch=a.constant?[]:[a];break;case s.Identifier:a.constant=!1;a.toWatch=[a];break;case s.MemberExpression:U(a.object,b);a.computed&&U(a.property,b);a.constant=a.object.constant&&
-(!a.computed||a.property.constant);a.toWatch=[a];break;case s.CallExpression:d=e=a.filter?!b(a.callee.name).$stateful:!1;c=[];q(a.arguments,function(a){U(a,b);d=d&&a.constant;a.constant||c.push.apply(c,a.toWatch)});a.constant=d;a.toWatch=e?c:[a];break;case s.AssignmentExpression:U(a.left,b);U(a.right,b);a.constant=a.left.constant&&a.right.constant;a.toWatch=[a];break;case s.ArrayExpression:d=!0;c=[];q(a.elements,function(a){U(a,b);d=d&&a.constant;a.constant||c.push.apply(c,a.toWatch)});a.constant=
-d;a.toWatch=c;break;case s.ObjectExpression:d=!0;c=[];q(a.properties,function(a){U(a.value,b);d=d&&a.value.constant&&!a.computed;a.value.constant||c.push.apply(c,a.value.toWatch);a.computed&&(U(a.key,b),a.key.constant||c.push.apply(c,a.key.toWatch))});a.constant=d;a.toWatch=c;break;case s.ThisExpression:a.constant=!1;a.toWatch=[];break;case s.LocalsExpression:a.constant=!1,a.toWatch=[]}}function Ed(a){if(1===a.length){a=a[0].expression;var b=a.toWatch;return 1!==b.length?b:b[0]!==a?b:void 0}}function Fd(a){return a.type===
-s.Identifier||a.type===s.MemberExpression}function Gd(a){if(1===a.body.length&&Fd(a.body[0].expression))return{type:s.AssignmentExpression,left:a.body[0].expression,right:{type:s.NGValueParameter},operator:"="}}function Hd(a){this.$filter=a}function Id(a){this.$filter=a}function uc(a,b,d){this.ast=new s(a,d);this.astCompiler=d.csp?new Id(b):new Hd(b)}function vc(a){return D(a.valueOf)?a.valueOf():Bg.call(a)}function Lf(){var a=V(),b={"true":!0,"false":!1,"null":null,undefined:void 0},d,c;this.addLiteral=
-function(a,c){b[a]=c};this.setIdentifierFns=function(a,b){d=a;c=b;return this};this.$get=["$filter",function(e){function f(a,b,c){return null==a||null==b?a===b:"object"!==typeof a||(a=vc(a),"object"!==typeof a||c)?a===b||a!==a&&b!==b:!1}function g(a,b,c,d,e){var g=d.inputs,h;if(1===g.length){var k=f,g=g[0];return a.$watch(function(a){var b=g(a);f(b,k,d.literal)||(h=d(a,void 0,void 0,[b]),k=b&&vc(b));return h},b,c,e)}for(var l=[],m=[],n=0,E=g.length;n<E;n++)l[n]=f,m[n]=null;return a.$watch(function(a){for(var b=
-!1,c=0,e=g.length;c<e;c++){var k=g[c](a);if(b||(b=!f(k,l[c],d.literal)))m[c]=k,l[c]=k&&vc(k)}b&&(h=d(a,void 0,void 0,m));return h},b,c,e)}function h(a,b,c,d,e){function f(a){return d(a)}function h(a,c,d){n=a;D(b)&&b(a,c,d);l(a)&&d.$$postDigest(function(){l(n)&&m()})}var l=d.literal?k:u,m,n;return m=d.inputs?g(a,h,c,d,e):a.$watch(f,h,c)}function k(a){var b=!0;q(a,function(a){u(a)||(b=!1)});return b}function l(a,b,c,d){var e=a.$watch(function(a){e();return d(a)},b,c);return e}function m(a,b){function c(d,
-e,g,h){g=f&&h?h[0]:a(d,e,g,h);return b(g,d,e)}function d(c,e,g,k){g=f&&k?k[0]:a(c,e,g,k);c=b(g,c,e);return h(g)?c:g}if(!b)return a;var e=a.$$watchDelegate,f=!1,h=a.literal?k:u,l=a.oneTime?d:c;l.literal=a.literal;l.oneTime=a.oneTime;f=!a.inputs;e&&e!==g?(l.$$watchDelegate=e,l.inputs=a.inputs):b.$stateful||(l.$$watchDelegate=g,l.inputs=a.inputs?a.inputs:[a]);return l}var n={csp:Ga().noUnsafeEval,literals:ra(b),isIdentifierStart:D(d)&&d,isIdentifierContinue:D(c)&&c};return function(b,c){var d,f,k;switch(typeof b){case "string":return k=
-b=b.trim(),d=a[k],d||(":"===b.charAt(0)&&":"===b.charAt(1)&&(f=!0,b=b.substring(2)),d=new wc(n),d=(new uc(d,e,n)).parse(b),d.constant?d.$$watchDelegate=l:f?(d.oneTime=!0,d.$$watchDelegate=h):d.inputs&&(d.$$watchDelegate=g),a[k]=d),m(d,c);case "function":return m(b,c);default:return m(z,c)}}}]}function Nf(){var a=!0;this.$get=["$rootScope","$exceptionHandler",function(b,d){return Jd(function(a){b.$evalAsync(a)},d,a)}];this.errorOnUnhandledRejections=function(b){return u(b)?(a=b,this):a}}function Of(){var a=
-!0;this.$get=["$browser","$exceptionHandler",function(b,d){return Jd(function(a){b.defer(a)},d,a)}];this.errorOnUnhandledRejections=function(b){return u(b)?(a=b,this):a}}function Jd(a,b,d){function c(){return new e}function e(){var a=this.promise=new f;this.resolve=function(b){k(a,b)};this.reject=function(b){m(a,b)};this.notify=function(b){p(a,b)}}function f(){this.$$state={status:0}}function g(){for(;!s&&A.length;){var a=A.shift();if(!a.pur){a.pur=!0;var c=a.value,c="Possibly unhandled rejection: "+
-("function"===typeof c?c.toString().replace(/ \{[\s\S]*$/,""):w(c)?"undefined":"string"!==typeof c?Be(c,void 0):c);a.value instanceof Error?b(a.value,c):b(c)}}}function h(b){!d||b.pending||2!==b.status||b.pur||(0===s&&0===A.length&&a(g),A.push(b));!b.processScheduled&&b.pending&&(b.processScheduled=!0,++s,a(function(){var c,e,f;f=b.pending;b.processScheduled=!1;b.pending=void 0;try{for(var h=0,l=f.length;h<l;++h){b.pur=!0;e=f[h][0];c=f[h][b.status];try{D(c)?k(e,c(b.value)):1===b.status?k(e,b.value):
-m(e,b.value)}catch(n){m(e,n)}}}finally{--s,d&&0===s&&a(g)}}))}function k(a,b){a.$$state.status||(b===a?n(a,M("qcycle",b)):l(a,b))}function l(a,b){function c(b){g||(g=!0,l(a,b))}function d(b){g||(g=!0,n(a,b))}function e(b){p(a,b)}var f,g=!1;try{if(C(b)||D(b))f=b.then;D(f)?(a.$$state.status=-1,f.call(b,c,d,e)):(a.$$state.value=b,a.$$state.status=1,h(a.$$state))}catch(k){d(k)}}function m(a,b){a.$$state.status||n(a,b)}function n(a,b){a.$$state.value=b;a.$$state.status=2;h(a.$$state)}function p(c,d){var e=
-c.$$state.pending;0>=c.$$state.status&&e&&e.length&&a(function(){for(var a,c,f=0,g=e.length;f<g;f++){c=e[f][0];a=e[f][3];try{p(c,D(a)?a(d):d)}catch(h){b(h)}}})}function r(a){var b=new f;m(b,a);return b}function J(a,b,c){var d=null;try{D(c)&&(d=c())}catch(e){return r(e)}return d&&D(d.then)?d.then(function(){return b(a)},r):b(a)}function v(a,b,c,d){var e=new f;k(e,a);return e.then(b,c,d)}function t(a){if(!D(a))throw M("norslvr",a);var b=new f;a(function(a){k(b,a)},function(a){m(b,a)});return b}var M=
-L("$q",TypeError),s=0,A=[];S(f.prototype,{then:function(a,b,c){if(w(a)&&w(b)&&w(c))return this;var d=new f;this.$$state.pending=this.$$state.pending||[];this.$$state.pending.push([d,a,b,c]);0<this.$$state.status&&h(this.$$state);return d},"catch":function(a){return this.then(null,a)},"finally":function(a,b){return this.then(function(b){return J(b,u,a)},function(b){return J(b,r,a)},b)}});var u=v;t.prototype=f.prototype;t.defer=c;t.reject=r;t.when=v;t.resolve=u;t.all=function(a){var b=new f,c=0,d=H(a)?
-[]:{};q(a,function(a,e){c++;v(a).then(function(a){d[e]=a;--c||k(b,d)},function(a){m(b,a)})});0===c&&k(b,d);return b};t.race=function(a){var b=c();q(a,function(a){v(a).then(b.resolve,b.reject)});return b.promise};return t}function Xf(){this.$get=["$window","$timeout",function(a,b){var d=a.requestAnimationFrame||a.webkitRequestAnimationFrame,c=a.cancelAnimationFrame||a.webkitCancelAnimationFrame||a.webkitCancelRequestAnimationFrame,e=!!d,f=e?function(a){var b=d(a);return function(){c(b)}}:function(a){var c=
-b(a,16.66,!1);return function(){b.cancel(c)}};f.supported=e;return f}]}function Mf(){function a(a){function b(){this.$$watchers=this.$$nextSibling=this.$$childHead=this.$$childTail=null;this.$$listeners={};this.$$listenerCount={};this.$$watchersCount=0;this.$id=++qb;this.$$ChildScope=null}b.prototype=a;return b}var b=10,d=L("$rootScope"),c=null,e=null;this.digestTtl=function(a){arguments.length&&(b=a);return b};this.$get=["$exceptionHandler","$parse","$browser",function(f,g,h){function k(a){a.currentScope.$$destroyed=
-!0}function l(a){9===za&&(a.$$childHead&&l(a.$$childHead),a.$$nextSibling&&l(a.$$nextSibling));a.$parent=a.$$nextSibling=a.$$prevSibling=a.$$childHead=a.$$childTail=a.$root=a.$$watchers=null}function m(){this.$id=++qb;this.$$phase=this.$parent=this.$$watchers=this.$$nextSibling=this.$$prevSibling=this.$$childHead=this.$$childTail=null;this.$root=this;this.$$destroyed=!1;this.$$listeners={};this.$$listenerCount={};this.$$watchersCount=0;this.$$isolateBindings=null}function n(a){if(M.$$phase)throw d("inprog",
-M.$$phase);M.$$phase=a}function p(a,b){do a.$$watchersCount+=b;while(a=a.$parent)}function r(a,b,c){do a.$$listenerCount[c]-=b,0===a.$$listenerCount[c]&&delete a.$$listenerCount[c];while(a=a.$parent)}function J(){}function v(){for(;u.length;)try{u.shift()()}catch(a){f(a)}e=null}function t(){null===e&&(e=h.defer(function(){M.$apply(v)}))}m.prototype={constructor:m,$new:function(b,c){var d;c=c||this;b?(d=new m,d.$root=this.$root):(this.$$ChildScope||(this.$$ChildScope=a(this)),d=new this.$$ChildScope);
-d.$parent=c;d.$$prevSibling=c.$$childTail;c.$$childHead?(c.$$childTail.$$nextSibling=d,c.$$childTail=d):c.$$childHead=c.$$childTail=d;(b||c!==this)&&d.$on("$destroy",k);return d},$watch:function(a,b,d,e){var f=g(a);if(f.$$watchDelegate)return f.$$watchDelegate(this,b,d,f,a);var h=this,k=h.$$watchers,l={fn:b,last:J,get:f,exp:e||a,eq:!!d};c=null;D(b)||(l.fn=z);k||(k=h.$$watchers=[],k.$$digestWatchIndex=-1);k.unshift(l);k.$$digestWatchIndex++;p(this,1);return function(){var a=$a(k,l);0<=a&&(p(h,-1),
-a<k.$$digestWatchIndex&&k.$$digestWatchIndex--);c=null}},$watchGroup:function(a,b){function c(){h=!1;k?(k=!1,b(e,e,g)):b(e,d,g)}var d=Array(a.length),e=Array(a.length),f=[],g=this,h=!1,k=!0;if(!a.length){var l=!0;g.$evalAsync(function(){l&&b(e,e,g)});return function(){l=!1}}if(1===a.length)return this.$watch(a[0],function(a,c,f){e[0]=a;d[0]=c;b(e,a===c?e:d,f)});q(a,function(a,b){var k=g.$watch(a,function(a,f){e[b]=a;d[b]=f;h||(h=!0,g.$evalAsync(c))});f.push(k)});return function(){for(;f.length;)f.shift()()}},
-$watchCollection:function(a,b){function c(a){e=a;var b,d,g,h;if(!w(e)){if(C(e))if(qa(e))for(f!==n&&(f=n,t=f.length=0,l++),a=e.length,t!==a&&(l++,f.length=t=a),b=0;b<a;b++)h=f[b],g=e[b],d=h!==h&&g!==g,d||h===g||(l++,f[b]=g);else{f!==p&&(f=p={},t=0,l++);a=0;for(b in e)ua.call(e,b)&&(a++,g=e[b],h=f[b],b in f?(d=h!==h&&g!==g,d||h===g||(l++,f[b]=g)):(t++,f[b]=g,l++));if(t>a)for(b in l++,f)ua.call(e,b)||(t--,delete f[b])}else f!==e&&(f=e,l++);return l}}c.$stateful=!0;var d=this,e,f,h,k=1<b.length,l=0,m=
-g(a,c),n=[],p={},r=!0,t=0;return this.$watch(m,function(){r?(r=!1,b(e,e,d)):b(e,h,d);if(k)if(C(e))if(qa(e)){h=Array(e.length);for(var a=0;a<e.length;a++)h[a]=e[a]}else for(a in h={},e)ua.call(e,a)&&(h[a]=e[a]);else h=e})},$digest:function(){var a,g,k,l,m,p,r,t=b,q,u=[],w,x;n("$digest");h.$$checkUrlChange();this===M&&null!==e&&(h.defer.cancel(e),v());c=null;do{r=!1;q=this;for(p=0;p<s.length;p++){try{x=s[p],l=x.fn,l(x.scope,x.locals)}catch(z){f(z)}c=null}s.length=0;a:do{if(p=q.$$watchers)for(p.$$digestWatchIndex=
-p.length;p.$$digestWatchIndex--;)try{if(a=p[p.$$digestWatchIndex])if(m=a.get,(g=m(q))!==(k=a.last)&&!(a.eq?sa(g,k):da(g)&&da(k)))r=!0,c=a,a.last=a.eq?ra(g,null):g,l=a.fn,l(g,k===J?g:k,q),5>t&&(w=4-t,u[w]||(u[w]=[]),u[w].push({msg:D(a.exp)?"fn: "+(a.exp.name||a.exp.toString()):a.exp,newVal:g,oldVal:k}));else if(a===c){r=!1;break a}}catch(B){f(B)}if(!(p=q.$$watchersCount&&q.$$childHead||q!==this&&q.$$nextSibling))for(;q!==this&&!(p=q.$$nextSibling);)q=q.$parent}while(q=p);if((r||s.length)&&!t--)throw M.$$phase=
-null,d("infdig",b,u);}while(r||s.length);for(M.$$phase=null;I<A.length;)try{A[I++]()}catch(F){f(F)}A.length=I=0;h.$$checkUrlChange()},$destroy:function(){if(!this.$$destroyed){var a=this.$parent;this.$broadcast("$destroy");this.$$destroyed=!0;this===M&&h.$$applicationDestroyed();p(this,-this.$$watchersCount);for(var b in this.$$listenerCount)r(this,this.$$listenerCount[b],b);a&&a.$$childHead===this&&(a.$$childHead=this.$$nextSibling);a&&a.$$childTail===this&&(a.$$childTail=this.$$prevSibling);this.$$prevSibling&&
-(this.$$prevSibling.$$nextSibling=this.$$nextSibling);this.$$nextSibling&&(this.$$nextSibling.$$prevSibling=this.$$prevSibling);this.$destroy=this.$digest=this.$apply=this.$evalAsync=this.$applyAsync=z;this.$on=this.$watch=this.$watchGroup=function(){return z};this.$$listeners={};this.$$nextSibling=null;l(this)}},$eval:function(a,b){return g(a)(this,b)},$evalAsync:function(a,b){M.$$phase||s.length||h.defer(function(){s.length&&M.$digest()});s.push({scope:this,fn:g(a),locals:b})},$$postDigest:function(a){A.push(a)},
-$apply:function(a){try{n("$apply");try{return this.$eval(a)}finally{M.$$phase=null}}catch(b){f(b)}finally{try{M.$digest()}catch(c){throw f(c),c;}}},$applyAsync:function(a){function b(){c.$eval(a)}var c=this;a&&u.push(b);a=g(a);t()},$on:function(a,b){var c=this.$$listeners[a];c||(this.$$listeners[a]=c=[]);c.push(b);var d=this;do d.$$listenerCount[a]||(d.$$listenerCount[a]=0),d.$$listenerCount[a]++;while(d=d.$parent);var e=this;return function(){var d=c.indexOf(b);-1!==d&&(c[d]=null,r(e,1,a))}},$emit:function(a,
-b){var c=[],d,e=this,g=!1,h={name:a,targetScope:e,stopPropagation:function(){g=!0},preventDefault:function(){h.defaultPrevented=!0},defaultPrevented:!1},k=ab([h],arguments,1),l,m;do{d=e.$$listeners[a]||c;h.currentScope=e;l=0;for(m=d.length;l<m;l++)if(d[l])try{d[l].apply(null,k)}catch(n){f(n)}else d.splice(l,1),l--,m--;if(g)return h.currentScope=null,h;e=e.$parent}while(e);h.currentScope=null;return h},$broadcast:function(a,b){var c=this,d=this,e={name:a,targetScope:this,preventDefault:function(){e.defaultPrevented=
-!0},defaultPrevented:!1};if(!this.$$listenerCount[a])return e;for(var g=ab([e],arguments,1),h,k;c=d;){e.currentScope=c;d=c.$$listeners[a]||[];h=0;for(k=d.length;h<k;h++)if(d[h])try{d[h].apply(null,g)}catch(l){f(l)}else d.splice(h,1),h--,k--;if(!(d=c.$$listenerCount[a]&&c.$$childHead||c!==this&&c.$$nextSibling))for(;c!==this&&!(d=c.$$nextSibling);)c=c.$parent}e.currentScope=null;return e}};var M=new m,s=M.$$asyncQueue=[],A=M.$$postDigestQueue=[],u=M.$$applyAsyncQueue=[],I=0;return M}]}function Ee(){var a=
-/^\s*(https?|ftp|mailto|tel|file):/,b=/^\s*((https?|ftp|file|blob):|data:image\/)/;this.aHrefSanitizationWhitelist=function(b){return u(b)?(a=b,this):a};this.imgSrcSanitizationWhitelist=function(a){return u(a)?(b=a,this):b};this.$get=function(){return function(d,c){var e=c?b:a,f;f=Ca(d).href;return""===f||f.match(e)?d:"unsafe:"+f}}}function Cg(a){if("self"===a)return a;if(F(a)){if(-1<a.indexOf("***"))throw ta("iwcard",a);a=Kd(a).replace(/\\\*\\\*/g,".*").replace(/\\\*/g,"[^:/.?&;]*");return new RegExp("^"+
-a+"$")}if(Xa(a))return new RegExp("^"+a.source+"$");throw ta("imatcher");}function Ld(a){var b=[];u(a)&&q(a,function(a){b.push(Cg(a))});return b}function Qf(){this.SCE_CONTEXTS=oa;var a=["self"],b=[];this.resourceUrlWhitelist=function(b){arguments.length&&(a=Ld(b));return a};this.resourceUrlBlacklist=function(a){arguments.length&&(b=Ld(a));return b};this.$get=["$injector",function(d){function c(a,b){return"self"===a?yd(b):!!a.exec(b.href)}function e(a){var b=function(a){this.$$unwrapTrustedValue=
-function(){return a}};a&&(b.prototype=new a);b.prototype.valueOf=function(){return this.$$unwrapTrustedValue()};b.prototype.toString=function(){return this.$$unwrapTrustedValue().toString()};return b}var f=function(a){throw ta("unsafe");};d.has("$sanitize")&&(f=d.get("$sanitize"));var g=e(),h={};h[oa.HTML]=e(g);h[oa.CSS]=e(g);h[oa.URL]=e(g);h[oa.JS]=e(g);h[oa.RESOURCE_URL]=e(h[oa.URL]);return{trustAs:function(a,b){var c=h.hasOwnProperty(a)?h[a]:null;if(!c)throw ta("icontext",a,b);if(null===b||w(b)||
-""===b)return b;if("string"!==typeof b)throw ta("itype",a);return new c(b)},getTrusted:function(d,e){if(null===e||w(e)||""===e)return e;var g=h.hasOwnProperty(d)?h[d]:null;if(g&&e instanceof g)return e.$$unwrapTrustedValue();if(d===oa.RESOURCE_URL){var g=Ca(e.toString()),n,p,r=!1;n=0;for(p=a.length;n<p;n++)if(c(a[n],g)){r=!0;break}if(r)for(n=0,p=b.length;n<p;n++)if(c(b[n],g)){r=!1;break}if(r)return e;throw ta("insecurl",e.toString());}if(d===oa.HTML)return f(e);throw ta("unsafe");},valueOf:function(a){return a instanceof
-g?a.$$unwrapTrustedValue():a}}}]}function Pf(){var a=!0;this.enabled=function(b){arguments.length&&(a=!!b);return a};this.$get=["$parse","$sceDelegate",function(b,d){if(a&&8>za)throw ta("iequirks");var c=pa(oa);c.isEnabled=function(){return a};c.trustAs=d.trustAs;c.getTrusted=d.getTrusted;c.valueOf=d.valueOf;a||(c.trustAs=c.getTrusted=function(a,b){return b},c.valueOf=Ya);c.parseAs=function(a,d){var e=b(d);return e.literal&&e.constant?e:b(d,function(b){return c.getTrusted(a,b)})};var e=c.parseAs,
-f=c.getTrusted,g=c.trustAs;q(oa,function(a,b){var d=Q(b);c[("parse_as_"+d).replace(xc,gb)]=function(b){return e(a,b)};c[("get_trusted_"+d).replace(xc,gb)]=function(b){return f(a,b)};c[("trust_as_"+d).replace(xc,gb)]=function(b){return g(a,b)}});return c}]}function Rf(){this.$get=["$window","$document",function(a,b){var d={},c=!((!a.nw||!a.nw.process)&&a.chrome&&(a.chrome.app&&a.chrome.app.runtime||!a.chrome.app&&a.chrome.runtime&&a.chrome.runtime.id))&&a.history&&a.history.pushState,e=Z((/android (\d+)/.exec(Q((a.navigator||
-{}).userAgent))||[])[1]),f=/Boxee/i.test((a.navigator||{}).userAgent),g=b[0]||{},h=g.body&&g.body.style,k=!1,l=!1;h&&(k=!!("transition"in h||"webkitTransition"in h),l=!!("animation"in h||"webkitAnimation"in h));return{history:!(!c||4>e||f),hasEvent:function(a){if("input"===a&&za)return!1;if(w(d[a])){var b=g.createElement("div");d[a]="on"+a in b}return d[a]},csp:Ga(),transitions:k,animations:l,android:e}}]}function Tf(){var a;this.httpOptions=function(b){return b?(a=b,this):a};this.$get=["$exceptionHandler",
-"$templateCache","$http","$q","$sce",function(b,d,c,e,f){function g(h,k){g.totalPendingRequests++;if(!F(h)||w(d.get(h)))h=f.getTrustedResourceUrl(h);var l=c.defaults&&c.defaults.transformResponse;H(l)?l=l.filter(function(a){return a!==nc}):l===nc&&(l=null);return c.get(h,S({cache:d,transformResponse:l},a)).finally(function(){g.totalPendingRequests--}).then(function(a){d.put(h,a.data);return a.data},function(a){k||(a=Dg("tpload",h,a.status,a.statusText),b(a));return e.reject(a)})}g.totalPendingRequests=
-0;return g}]}function Uf(){this.$get=["$rootScope","$browser","$location",function(a,b,d){return{findBindings:function(a,b,d){a=a.getElementsByClassName("ng-binding");var g=[];q(a,function(a){var c=ea.element(a).data("$binding");c&&q(c,function(c){d?(new RegExp("(^|\\s)"+Kd(b)+"(\\s|\\||$)")).test(c)&&g.push(a):-1!==c.indexOf(b)&&g.push(a)})});return g},findModels:function(a,b,d){for(var g=["ng-","data-ng-","ng\\:"],h=0;h<g.length;++h){var k=a.querySelectorAll("["+g[h]+"model"+(d?"=":"*=")+'"'+b+
-'"]');if(k.length)return k}},getLocation:function(){return d.url()},setLocation:function(b){b!==d.url()&&(d.url(b),a.$digest())},whenStable:function(a){b.notifyWhenNoOutstandingRequests(a)}}}]}function Vf(){this.$get=["$rootScope","$browser","$q","$$q","$exceptionHandler",function(a,b,d,c,e){function f(f,k,l){D(f)||(l=k,k=f,f=z);var m=va.call(arguments,3),n=u(l)&&!l,p=(n?c:d).defer(),r=p.promise,q;q=b.defer(function(){try{p.resolve(f.apply(null,m))}catch(b){p.reject(b),e(b)}finally{delete g[r.$$timeoutId]}n||
-a.$apply()},k);r.$$timeoutId=q;g[q]=p;return r}var g={};f.cancel=function(a){return a&&a.$$timeoutId in g?(g[a.$$timeoutId].promise.catch(z),g[a.$$timeoutId].reject("canceled"),delete g[a.$$timeoutId],b.defer.cancel(a.$$timeoutId)):!1};return f}]}function Ca(a){za&&(aa.setAttribute("href",a),a=aa.href);aa.setAttribute("href",a);return{href:aa.href,protocol:aa.protocol?aa.protocol.replace(/:$/,""):"",host:aa.host,search:aa.search?aa.search.replace(/^\?/,""):"",hash:aa.hash?aa.hash.replace(/^#/,""):
-"",hostname:aa.hostname,port:aa.port,pathname:"/"===aa.pathname.charAt(0)?aa.pathname:"/"+aa.pathname}}function yd(a){a=F(a)?Ca(a):a;return a.protocol===Md.protocol&&a.host===Md.host}function Wf(){this.$get=la(x)}function Nd(a){function b(a){try{return decodeURIComponent(a)}catch(b){return a}}var d=a[0]||{},c={},e="";return function(){var a,g,h,k,l;try{a=d.cookie||""}catch(m){a=""}if(a!==e)for(e=a,a=e.split("; "),c={},h=0;h<a.length;h++)g=a[h],k=g.indexOf("="),0<k&&(l=b(g.substring(0,k)),w(c[l])&&
-(c[l]=b(g.substring(k+1))));return c}}function $f(){this.$get=Nd}function cd(a){function b(d,c){if(C(d)){var e={};q(d,function(a,c){e[c]=b(c,a)});return e}return a.factory(d+"Filter",c)}this.register=b;this.$get=["$injector",function(a){return function(b){return a.get(b+"Filter")}}];b("currency",Od);b("date",Pd);b("filter",Eg);b("json",Fg);b("limitTo",Gg);b("lowercase",Hg);b("number",Qd);b("orderBy",Rd);b("uppercase",Ig)}function Eg(){return function(a,b,d,c){if(!qa(a)){if(null==a)return a;throw L("filter")("notarray",
-a);}c=c||"$";var e;switch(yc(b)){case "function":break;case "boolean":case "null":case "number":case "string":e=!0;case "object":b=Jg(b,d,c,e);break;default:return a}return Array.prototype.filter.call(a,b)}}function Jg(a,b,d,c){var e=C(a)&&d in a;!0===b?b=sa:D(b)||(b=function(a,b){if(w(a))return!1;if(null===a||null===b)return a===b;if(C(b)||C(a)&&!Wb(a))return!1;a=Q(""+a);b=Q(""+b);return-1!==a.indexOf(b)});return function(f){return e&&!C(f)?Ea(f,a[d],b,d,!1):Ea(f,a,b,d,c)}}function Ea(a,b,d,c,e,
-f){var g=yc(a),h=yc(b);if("string"===h&&"!"===b.charAt(0))return!Ea(a,b.substring(1),d,c,e);if(H(a))return a.some(function(a){return Ea(a,b,d,c,e)});switch(g){case "object":var k;if(e){for(k in a)if(k.charAt&&"$"!==k.charAt(0)&&Ea(a[k],b,d,c,!0))return!0;return f?!1:Ea(a,b,d,c,!1)}if("object"===h){for(k in b)if(f=b[k],!D(f)&&!w(f)&&(g=k===c,!Ea(g?a:a[k],f,d,c,g,g)))return!1;return!0}return d(a,b);case "function":return!1;default:return d(a,b)}}function yc(a){return null===a?"null":typeof a}function Od(a){var b=
-a.NUMBER_FORMATS;return function(a,c,e){w(c)&&(c=b.CURRENCY_SYM);w(e)&&(e=b.PATTERNS[1].maxFrac);return null==a?a:Sd(a,b.PATTERNS[1],b.GROUP_SEP,b.DECIMAL_SEP,e).replace(/\u00A4/g,c)}}function Qd(a){var b=a.NUMBER_FORMATS;return function(a,c){return null==a?a:Sd(a,b.PATTERNS[0],b.GROUP_SEP,b.DECIMAL_SEP,c)}}function Kg(a){var b=0,d,c,e,f,g;-1<(c=a.indexOf(Td))&&(a=a.replace(Td,""));0<(e=a.search(/e/i))?(0>c&&(c=e),c+=+a.slice(e+1),a=a.substring(0,e)):0>c&&(c=a.length);for(e=0;a.charAt(e)===zc;e++);
-if(e===(g=a.length))d=[0],c=1;else{for(g--;a.charAt(g)===zc;)g--;c-=e;d=[];for(f=0;e<=g;e++,f++)d[f]=+a.charAt(e)}c>Ud&&(d=d.splice(0,Ud-1),b=c-1,c=1);return{d:d,e:b,i:c}}function Lg(a,b,d,c){var e=a.d,f=e.length-a.i;b=w(b)?Math.min(Math.max(d,f),c):+b;d=b+a.i;c=e[d];if(0<d){e.splice(Math.max(a.i,d));for(var g=d;g<e.length;g++)e[g]=0}else for(f=Math.max(0,f),a.i=1,e.length=Math.max(1,d=b+1),e[0]=0,g=1;g<d;g++)e[g]=0;if(5<=c)if(0>d-1){for(c=0;c>d;c--)e.unshift(0),a.i++;e.unshift(1);a.i++}else e[d-
-1]++;for(;f<Math.max(0,b);f++)e.push(0);if(b=e.reduceRight(function(a,b,c,d){b+=a;d[c]=b%10;return Math.floor(b/10)},0))e.unshift(b),a.i++}function Sd(a,b,d,c,e){if(!F(a)&&!ba(a)||isNaN(a))return"";var f=!isFinite(a),g=!1,h=Math.abs(a)+"",k="";if(f)k="\u221e";else{g=Kg(h);Lg(g,e,b.minFrac,b.maxFrac);k=g.d;h=g.i;e=g.e;f=[];for(g=k.reduce(function(a,b){return a&&!b},!0);0>h;)k.unshift(0),h++;0<h?f=k.splice(h,k.length):(f=k,k=[0]);h=[];for(k.length>=b.lgSize&&h.unshift(k.splice(-b.lgSize,k.length).join(""));k.length>
-b.gSize;)h.unshift(k.splice(-b.gSize,k.length).join(""));k.length&&h.unshift(k.join(""));k=h.join(d);f.length&&(k+=c+f.join(""));e&&(k+="e+"+e)}return 0>a&&!g?b.negPre+k+b.negSuf:b.posPre+k+b.posSuf}function Kb(a,b,d,c){var e="";if(0>a||c&&0>=a)c?a=-a+1:(a=-a,e="-");for(a=""+a;a.length<b;)a=zc+a;d&&(a=a.substr(a.length-b));return e+a}function Y(a,b,d,c,e){d=d||0;return function(f){f=f["get"+a]();if(0<d||f>-d)f+=d;0===f&&-12===d&&(f=12);return Kb(f,b,c,e)}}function mb(a,b,d){return function(c,e){var f=
-c["get"+a](),g=ub((d?"STANDALONE":"")+(b?"SHORT":"")+a);return e[g][f]}}function Vd(a){var b=(new Date(a,0,1)).getDay();return new Date(a,0,(4>=b?5:12)-b)}function Wd(a){return function(b){var d=Vd(b.getFullYear());b=+new Date(b.getFullYear(),b.getMonth(),b.getDate()+(4-b.getDay()))-+d;b=1+Math.round(b/6048E5);return Kb(b,a)}}function Ac(a,b){return 0>=a.getFullYear()?b.ERAS[0]:b.ERAS[1]}function Pd(a){function b(a){var b;if(b=a.match(d)){a=new Date(0);var f=0,g=0,h=b[8]?a.setUTCFullYear:a.setFullYear,
-k=b[8]?a.setUTCHours:a.setHours;b[9]&&(f=Z(b[9]+b[10]),g=Z(b[9]+b[11]));h.call(a,Z(b[1]),Z(b[2])-1,Z(b[3]));f=Z(b[4]||0)-f;g=Z(b[5]||0)-g;h=Z(b[6]||0);b=Math.round(1E3*parseFloat("0."+(b[7]||0)));k.call(a,f,g,h,b)}return a}var d=/^(\d{4})-?(\d\d)-?(\d\d)(?:T(\d\d)(?::?(\d\d)(?::?(\d\d)(?:\.(\d+))?)?)?(Z|([+-])(\d\d):?(\d\d))?)?$/;return function(c,d,f){var g="",h=[],k,l;d=d||"mediumDate";d=a.DATETIME_FORMATS[d]||d;F(c)&&(c=Mg.test(c)?Z(c):b(c));ba(c)&&(c=new Date(c));if(!ga(c)||!isFinite(c.getTime()))return c;
-for(;d;)(l=Ng.exec(d))?(h=ab(h,l,1),d=h.pop()):(h.push(d),d=null);var m=c.getTimezoneOffset();f&&(m=Pc(f,m),c=Yb(c,f,!0));q(h,function(b){k=Og[b];g+=k?k(c,a.DATETIME_FORMATS,m):"''"===b?"'":b.replace(/(^'|'$)/g,"").replace(/''/g,"'")});return g}}function Fg(){return function(a,b){w(b)&&(b=2);return cb(a,b)}}function Gg(){return function(a,b,d){b=Infinity===Math.abs(Number(b))?Number(b):Z(b);if(da(b))return a;ba(a)&&(a=a.toString());if(!qa(a))return a;d=!d||isNaN(d)?0:Z(d);d=0>d?Math.max(0,a.length+
-d):d;return 0<=b?Bc(a,d,d+b):0===d?Bc(a,b,a.length):Bc(a,Math.max(0,d+b),d)}}function Bc(a,b,d){return F(a)?a.slice(b,d):va.call(a,b,d)}function Rd(a){function b(b){return b.map(function(b){var c=1,d=Ya;if(D(b))d=b;else if(F(b)){if("+"===b.charAt(0)||"-"===b.charAt(0))c="-"===b.charAt(0)?-1:1,b=b.substring(1);if(""!==b&&(d=a(b),d.constant))var e=d(),d=function(a){return a[e]}}return{get:d,descending:c}})}function d(a){switch(typeof a){case "number":case "boolean":case "string":return!0;default:return!1}}
-function c(a,b){var c=0,d=a.type,k=b.type;if(d===k){var k=a.value,l=b.value;"string"===d?(k=k.toLowerCase(),l=l.toLowerCase()):"object"===d&&(C(k)&&(k=a.index),C(l)&&(l=b.index));k!==l&&(c=k<l?-1:1)}else c=d<k?-1:1;return c}return function(a,f,g,h){if(null==a)return a;if(!qa(a))throw L("orderBy")("notarray",a);H(f)||(f=[f]);0===f.length&&(f=["+"]);var k=b(f),l=g?-1:1,m=D(h)?h:c;a=Array.prototype.map.call(a,function(a,b){return{value:a,tieBreaker:{value:b,type:"number",index:b},predicateValues:k.map(function(c){var e=
-c.get(a);c=typeof e;if(null===e)c="string",e="null";else if("object"===c)a:{if(D(e.valueOf)&&(e=e.valueOf(),d(e)))break a;Wb(e)&&(e=e.toString(),d(e))}return{value:e,type:c,index:b}})}});a.sort(function(a,b){for(var c=0,d=k.length;c<d;c++){var e=m(a.predicateValues[c],b.predicateValues[c]);if(e)return e*k[c].descending*l}return m(a.tieBreaker,b.tieBreaker)*l});return a=a.map(function(a){return a.value})}}function Qa(a){D(a)&&(a={link:a});a.restrict=a.restrict||"AC";return la(a)}function Lb(a,b,d,
-c,e){this.$$controls=[];this.$error={};this.$$success={};this.$pending=void 0;this.$name=e(b.name||b.ngForm||"")(d);this.$dirty=!1;this.$valid=this.$pristine=!0;this.$submitted=this.$invalid=!1;this.$$parentForm=Mb;this.$$element=a;this.$$animate=c;Xd(this)}function Xd(a){a.$$classCache={};a.$$classCache[Yd]=!(a.$$classCache[nb]=a.$$element.hasClass(nb))}function Zd(a){function b(a,b,c){c&&!a.$$classCache[b]?(a.$$animate.addClass(a.$$element,b),a.$$classCache[b]=!0):!c&&a.$$classCache[b]&&(a.$$animate.removeClass(a.$$element,
-b),a.$$classCache[b]=!1)}function d(a,c,d){c=c?"-"+Tc(c,"-"):"";b(a,nb+c,!0===d);b(a,Yd+c,!1===d)}var c=a.set,e=a.unset;a.clazz.prototype.$setValidity=function(a,g,h){w(g)?(this.$pending||(this.$pending={}),c(this.$pending,a,h)):(this.$pending&&e(this.$pending,a,h),$d(this.$pending)&&(this.$pending=void 0));Ha(g)?g?(e(this.$error,a,h),c(this.$$success,a,h)):(c(this.$error,a,h),e(this.$$success,a,h)):(e(this.$error,a,h),e(this.$$success,a,h));this.$pending?(b(this,"ng-pending",!0),this.$valid=this.$invalid=
-void 0,d(this,"",null)):(b(this,"ng-pending",!1),this.$valid=$d(this.$error),this.$invalid=!this.$valid,d(this,"",this.$valid));g=this.$pending&&this.$pending[a]?void 0:this.$error[a]?!1:this.$$success[a]?!0:null;d(this,a,g);this.$$parentForm.$setValidity(a,g,this)}}function $d(a){if(a)for(var b in a)if(
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[51/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
HDFS-13258. Ozone: restructure Hdsl/Ozone code to separated maven subprojects.
Contributed by Elek Marton, Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain, Nanda Kumar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ce23d9ad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ce23d9ad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ce23d9ad
Branch: refs/heads/HDFS-7240
Commit: ce23d9adf004358013825f2a1ec684f35a953b4a
Parents: a5dfae6
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Mar 15 09:23:33 2018 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Mar 15 09:23:33 2018 -0700
----------------------------------------------------------------------
dev-support/bin/dist-layout-stitching | 19 +-
hadoop-cblock/pom.xml | 56 +
.../server/dev-support/findbugsExcludeFile.xml | 21 +
hadoop-cblock/server/pom.xml | 169 +++
.../apache/hadoop/cblock/CBlockConfigKeys.java | 222 +++
.../org/apache/hadoop/cblock/CBlockManager.java | 426 ++++++
.../org/apache/hadoop/cblock/CblockUtils.java | 129 ++
...ckServiceProtocolClientSideTranslatorPB.java | 135 ++
.../cblock/client/CBlockVolumeClient.java | 83 ++
.../hadoop/cblock/client/package-info.java | 18 +
.../cblock/exception/CBlockException.java | 29 +
.../hadoop/cblock/exception/package-info.java | 18 +
.../cblock/jscsiHelper/BlockWriterTask.java | 175 +++
...ockClientProtocolClientSideTranslatorPB.java | 147 ++
.../cblock/jscsiHelper/CBlockIStorageImpl.java | 440 ++++++
.../jscsiHelper/CBlockManagerHandler.java | 50 +
.../cblock/jscsiHelper/CBlockTargetMetrics.java | 334 +++++
.../cblock/jscsiHelper/CBlockTargetServer.java | 128 ++
.../jscsiHelper/ContainerCacheFlusher.java | 599 ++++++++
.../cblock/jscsiHelper/SCSITargetDaemon.java | 132 ++
.../cblock/jscsiHelper/cache/CacheModule.java | 52 +
.../cblock/jscsiHelper/cache/LogicalBlock.java | 50 +
.../cache/impl/AsyncBlockWriter.java | 221 +++
.../cache/impl/BlockBufferFlushTask.java | 118 ++
.../cache/impl/BlockBufferManager.java | 184 +++
.../cache/impl/CBlockLocalCache.java | 577 ++++++++
.../jscsiHelper/cache/impl/DiskBlock.java | 77 ++
.../jscsiHelper/cache/impl/SyncBlockReader.java | 263 ++++
.../jscsiHelper/cache/impl/package-info.java | 18 +
.../cblock/jscsiHelper/cache/package-info.java | 18 +
.../hadoop/cblock/jscsiHelper/package-info.java | 18 +
.../cblock/kubernetes/DynamicProvisioner.java | 331 +++++
.../hadoop/cblock/kubernetes/package-info.java | 23 +
.../hadoop/cblock/meta/ContainerDescriptor.java | 107 ++
.../hadoop/cblock/meta/VolumeDescriptor.java | 269 ++++
.../apache/hadoop/cblock/meta/VolumeInfo.java | 79 ++
.../apache/hadoop/cblock/meta/package-info.java | 18 +
.../org/apache/hadoop/cblock/package-info.java | 18 +
.../cblock/proto/CBlockClientProtocol.java | 38 +
.../cblock/proto/CBlockServiceProtocol.java | 45 +
.../cblock/proto/MountVolumeResponse.java | 79 ++
.../hadoop/cblock/proto/package-info.java | 18 +
.../CBlockClientServerProtocolPB.java | 37 +
...entServerProtocolServerSideTranslatorPB.java | 116 ++
.../protocolPB/CBlockServiceProtocolPB.java | 35 +
...ckServiceProtocolServerSideTranslatorPB.java | 159 +++
.../hadoop/cblock/protocolPB/package-info.java | 18 +
.../hadoop/cblock/storage/StorageManager.java | 427 ++++++
.../hadoop/cblock/storage/package-info.java | 18 +
.../org/apache/hadoop/cblock/util/KeyUtil.java | 49 +
.../apache/hadoop/cblock/util/package-info.java | 18 +
.../main/proto/CBlockClientServerProtocol.proto | 93 ++
.../src/main/proto/CBlockServiceProtocol.proto | 133 ++
.../src/main/resources/cblock-default.xml | 347 +++++
.../apache/hadoop/cblock/TestBufferManager.java | 456 ++++++
.../cblock/TestCBlockConfigurationFields.java | 35 +
.../hadoop/cblock/TestCBlockReadWrite.java | 377 +++++
.../apache/hadoop/cblock/TestCBlockServer.java | 212 +++
.../cblock/TestCBlockServerPersistence.java | 132 ++
.../hadoop/cblock/TestLocalBlockCache.java | 444 ++++++
.../kubernetes/TestDynamicProvisioner.java | 74 +
.../cblock/util/ContainerLookUpService.java | 73 +
.../hadoop/cblock/util/MockStorageClient.java | 176 +++
.../dynamicprovisioner/expected1-pv.json | 37 +
.../dynamicprovisioner/input1-pvc.json | 38 +
hadoop-cblock/tools/pom.xml | 42 +
.../org/apache/hadoop/cblock/cli/CBlockCli.java | 265 ++++
.../apache/hadoop/cblock/cli/package-info.java | 18 +
.../org/apache/hadoop/cblock/TestCBlockCLI.java | 242 ++++
.../src/main/bin/hadoop-functions.sh | 7 +
.../org/apache/hadoop/conf/ConfServlet.java | 103 +-
.../apache/hadoop/conf/OzoneConfiguration.java | 155 ---
.../java/org/apache/hadoop/fs/FileUtil.java | 49 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 2 +-
hadoop-dist/pom.xml | 38 +
.../src/main/compose/cblock/docker-compose.yaml | 20 +-
.../src/main/compose/cblock/docker-config | 2 +-
.../src/main/compose/ozone/docker-compose.yaml | 16 +-
.../src/main/compose/ozone/docker-config | 1 +
.../dev-support/findbugsExcludeFile.xml | 8 +-
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 41 +-
.../apache/hadoop/cblock/CBlockConfigKeys.java | 221 ---
.../org/apache/hadoop/cblock/package-info.java | 22 -
.../apache/hadoop/hdfs/protocol/DatanodeID.java | 26 +-
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 14 +-
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 231 ----
.../apache/hadoop/ozone/OzoneConfigKeys.java | 245 ----
.../org/apache/hadoop/ozone/OzoneConsts.java | 162 ---
.../apache/hadoop/ozone/client/BucketArgs.java | 123 --
.../apache/hadoop/ozone/client/ObjectStore.java | 210 ---
.../apache/hadoop/ozone/client/OzoneBucket.java | 360 -----
.../apache/hadoop/ozone/client/OzoneClient.java | 103 --
.../hadoop/ozone/client/OzoneClientFactory.java | 308 -----
.../client/OzoneClientInvocationHandler.java | 62 -
.../hadoop/ozone/client/OzoneClientUtils.java | 874 ------------
.../apache/hadoop/ozone/client/OzoneKey.java | 119 --
.../apache/hadoop/ozone/client/OzoneQuota.java | 203 ---
.../apache/hadoop/ozone/client/OzoneVolume.java | 293 ----
.../hadoop/ozone/client/ReplicationFactor.java | 63 -
.../hadoop/ozone/client/ReplicationType.java | 28 -
.../apache/hadoop/ozone/client/VolumeArgs.java | 128 --
.../ozone/client/io/ChunkGroupInputStream.java | 319 -----
.../ozone/client/io/ChunkGroupOutputStream.java | 492 -------
.../ozone/client/io/LengthInputStream.java | 49 -
.../client/io/OzoneContainerTranslation.java | 51 -
.../ozone/client/io/OzoneInputStream.java | 60 -
.../ozone/client/io/OzoneOutputStream.java | 64 -
.../hadoop/ozone/client/io/package-info.java | 23 -
.../hadoop/ozone/client/package-info.java | 23 -
.../ozone/client/protocol/ClientProtocol.java | 323 -----
.../ozone/client/protocol/package-info.java | 23 -
.../client/rest/DefaultRestServerSelector.java | 36 -
.../ozone/client/rest/OzoneException.java | 267 ----
.../ozone/client/rest/OzoneExceptionMapper.java | 45 -
.../hadoop/ozone/client/rest/RestClient.java | 799 -----------
.../ozone/client/rest/RestServerSelector.java | 40 -
.../client/rest/exceptions/package-info.java | 22 -
.../ozone/client/rest/headers/Header.java | 71 -
.../ozone/client/rest/headers/package-info.java | 22 -
.../hadoop/ozone/client/rest/package-info.java | 23 -
.../ozone/client/rest/response/BucketInfo.java | 230 ---
.../ozone/client/rest/response/KeyInfo.java | 218 ---
.../ozone/client/rest/response/VolumeInfo.java | 215 ---
.../ozone/client/rest/response/VolumeOwner.java | 60 -
.../client/rest/response/package-info.java | 24 -
.../hadoop/ozone/client/rpc/RpcClient.java | 567 --------
.../hadoop/ozone/client/rpc/package-info.java | 23 -
.../apache/hadoop/ozone/common/BlockGroup.java | 87 --
.../ozone/common/DeleteBlockGroupResult.java | 94 --
.../hadoop/ozone/common/package-info.java | 21 -
.../InvalidStateTransitionException.java | 42 -
.../ozone/common/statemachine/StateMachine.java | 68 -
.../ozone/common/statemachine/package-info.java | 21 -
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 --
.../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ----
.../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 ----
.../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 --
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 243 ----
.../ozone/ksm/helpers/KsmKeyLocationInfo.java | 136 --
.../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 --
.../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 --
.../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 ---
.../ozone/ksm/helpers/OpenKeySession.java | 50 -
.../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 ----
.../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 --
.../hadoop/ozone/ksm/helpers/package-info.java | 18 -
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../ksm/protocol/KeySpaceManagerProtocol.java | 245 ----
.../hadoop/ozone/ksm/protocol/package-info.java | 19 -
...ceManagerProtocolClientSideTranslatorPB.java | 744 ----------
.../protocolPB/KeySpaceManagerProtocolPB.java | 34 -
.../ozone/ksm/protocolPB/package-info.java | 19 -
.../org/apache/hadoop/ozone/package-info.java | 22 -
.../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 --
.../hadoop/ozone/protocolPB/package-info.java | 24 -
.../container/ContainerStates/ContainerID.java | 97 --
.../container/ContainerStates/package-info.java | 22 -
.../org/apache/hadoop/scm/ScmConfigKeys.java | 260 ----
.../java/org/apache/hadoop/scm/ScmInfo.java | 81 --
.../org/apache/hadoop/scm/XceiverClient.java | 192 ---
.../apache/hadoop/scm/XceiverClientHandler.java | 204 ---
.../hadoop/scm/XceiverClientInitializer.java | 72 -
.../apache/hadoop/scm/XceiverClientManager.java | 219 ---
.../apache/hadoop/scm/XceiverClientMetrics.java | 92 --
.../apache/hadoop/scm/XceiverClientRatis.java | 264 ----
.../org/apache/hadoop/scm/XceiverClientSpi.java | 129 --
.../scm/client/ContainerOperationClient.java | 403 ------
.../org/apache/hadoop/scm/client/ScmClient.java | 139 --
.../apache/hadoop/scm/client/package-info.java | 24 -
.../common/helpers/AllocatedBlock.java | 77 --
.../container/common/helpers/ContainerInfo.java | 333 -----
.../common/helpers/DeleteBlockResult.java | 52 -
.../scm/container/common/helpers/Pipeline.java | 253 ----
.../common/helpers/PipelineChannel.java | 121 --
.../helpers/StorageContainerException.java | 104 --
.../container/common/helpers/package-info.java | 22 -
.../org/apache/hadoop/scm/package-info.java | 24 -
.../hadoop/scm/protocol/LocatedContainer.java | 127 --
.../scm/protocol/ScmBlockLocationProtocol.java | 72 -
.../hadoop/scm/protocol/ScmLocatedBlock.java | 100 --
.../StorageContainerLocationProtocol.java | 123 --
.../hadoop/scm/protocol/package-info.java | 19 -
...kLocationProtocolClientSideTranslatorPB.java | 207 ---
.../protocolPB/ScmBlockLocationProtocolPB.java | 35 -
...rLocationProtocolClientSideTranslatorPB.java | 304 ----
.../StorageContainerLocationProtocolPB.java | 34 -
.../hadoop/scm/protocolPB/package-info.java | 24 -
.../hadoop/scm/storage/ChunkInputStream.java | 261 ----
.../hadoop/scm/storage/ChunkOutputStream.java | 227 ---
.../scm/storage/ContainerProtocolCalls.java | 397 ------
.../apache/hadoop/scm/storage/package-info.java | 23 -
.../main/java/org/apache/ratis/RatisHelper.java | 128 --
.../java/org/apache/ratis/package-info.java | 22 -
.../com/google/protobuf/ShadedProtoUtil.java | 38 -
.../com/google/protobuf/package-info.java | 22 -
.../main/proto/DatanodeContainerProtocol.proto | 415 ------
.../main/proto/KeySpaceManagerProtocol.proto | 457 ------
.../src/main/proto/Ozone.proto | 161 ---
.../main/proto/ScmBlockLocationProtocol.proto | 166 ---
.../StorageContainerLocationProtocol.proto | 214 ---
.../src/main/proto/hdfs.proto | 1 +
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 141 --
.../hadoop/ozone/common/TestStateMachine.java | 104 --
.../java/org/apache/hadoop/scm/TestArchive.java | 114 --
.../org/apache/hadoop/scm/package-info.java | 21 -
.../dev-support/findbugsExcludeFile.xml | 9 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 124 +-
.../hadoop-hdfs/src/main/bin/hdfs | 47 -
.../hadoop-hdfs/src/main/bin/start-balancer.sh | 4 -
.../hadoop-hdfs/src/main/bin/start-ozone.sh | 105 --
.../hadoop-hdfs/src/main/bin/stop-balancer.sh | 4 -
.../hadoop-hdfs/src/main/bin/stop-ozone.sh | 99 --
.../hadoop-hdfs/src/main/conf/ozone-site.xml | 24 -
.../org/apache/hadoop/cblock/CBlockManager.java | 421 ------
.../org/apache/hadoop/cblock/cli/CBlockCli.java | 289 ----
.../apache/hadoop/cblock/cli/package-info.java | 18 -
...ckServiceProtocolClientSideTranslatorPB.java | 135 --
.../cblock/client/CBlockVolumeClient.java | 82 --
.../hadoop/cblock/client/package-info.java | 18 -
.../cblock/exception/CBlockException.java | 29 -
.../hadoop/cblock/exception/package-info.java | 18 -
.../cblock/jscsiHelper/BlockWriterTask.java | 175 ---
...ockClientProtocolClientSideTranslatorPB.java | 147 --
.../cblock/jscsiHelper/CBlockIStorageImpl.java | 440 ------
.../jscsiHelper/CBlockManagerHandler.java | 50 -
.../cblock/jscsiHelper/CBlockTargetMetrics.java | 334 -----
.../cblock/jscsiHelper/CBlockTargetServer.java | 128 --
.../jscsiHelper/ContainerCacheFlusher.java | 599 --------
.../cblock/jscsiHelper/SCSITargetDaemon.java | 129 --
.../cblock/jscsiHelper/cache/CacheModule.java | 52 -
.../cblock/jscsiHelper/cache/LogicalBlock.java | 50 -
.../cache/impl/AsyncBlockWriter.java | 221 ---
.../cache/impl/BlockBufferFlushTask.java | 118 --
.../cache/impl/BlockBufferManager.java | 184 ---
.../cache/impl/CBlockLocalCache.java | 577 --------
.../jscsiHelper/cache/impl/DiskBlock.java | 77 --
.../jscsiHelper/cache/impl/SyncBlockReader.java | 263 ----
.../jscsiHelper/cache/impl/package-info.java | 18 -
.../cblock/jscsiHelper/cache/package-info.java | 18 -
.../hadoop/cblock/jscsiHelper/package-info.java | 18 -
.../cblock/kubernetes/DynamicProvisioner.java | 330 -----
.../hadoop/cblock/kubernetes/package-info.java | 23 -
.../hadoop/cblock/meta/ContainerDescriptor.java | 107 --
.../hadoop/cblock/meta/VolumeDescriptor.java | 269 ----
.../apache/hadoop/cblock/meta/VolumeInfo.java | 79 --
.../apache/hadoop/cblock/meta/package-info.java | 18 -
.../org/apache/hadoop/cblock/package-info.java | 18 -
.../cblock/proto/CBlockClientProtocol.java | 38 -
.../cblock/proto/CBlockServiceProtocol.java | 45 -
.../cblock/proto/MountVolumeResponse.java | 79 --
.../hadoop/cblock/proto/package-info.java | 18 -
.../CBlockClientServerProtocolPB.java | 37 -
...entServerProtocolServerSideTranslatorPB.java | 116 --
.../protocolPB/CBlockServiceProtocolPB.java | 35 -
...ckServiceProtocolServerSideTranslatorPB.java | 159 ---
.../hadoop/cblock/protocolPB/package-info.java | 18 -
.../hadoop/cblock/storage/StorageManager.java | 427 ------
.../hadoop/cblock/storage/package-info.java | 18 -
.../org/apache/hadoop/cblock/util/KeyUtil.java | 49 -
.../apache/hadoop/cblock/util/package-info.java | 18 -
.../java/org/apache/hadoop/hdfs/DFSUtil.java | 20 -
.../hdfs/server/common/HdfsServerConstants.java | 7 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 132 +-
.../server/datanode/DataNodeServicePlugin.java | 48 +
.../server/datanode/ObjectStoreHandler.java | 189 ---
.../datanode/fsdataset/impl/FsDatasetImpl.java | 2 +-
.../server/datanode/web/DatanodeHttpServer.java | 19 +-
.../web/RestCsrfPreventionFilterHandler.java | 2 +-
.../hdfs/server/datanode/web/URLDispatcher.java | 76 +-
.../hdfs/server/protocol/NamespaceInfo.java | 12 +-
.../org/apache/hadoop/hdfs/tools/GetConf.java | 49 +-
.../InconsistentStorageStateException.java | 51 -
.../org/apache/hadoop/ozone/common/Storage.java | 249 ----
.../apache/hadoop/ozone/common/StorageInfo.java | 184 ---
.../hadoop/ozone/common/package-info.java | 18 -
.../container/common/helpers/ChunkInfo.java | 185 ---
.../container/common/helpers/ChunkUtils.java | 345 -----
.../container/common/helpers/ContainerData.java | 326 -----
.../common/helpers/ContainerMetrics.java | 121 --
.../common/helpers/ContainerReport.java | 218 ---
.../common/helpers/ContainerUtils.java | 477 -------
.../helpers/DeletedContainerBlocksSummary.java | 103 --
.../container/common/helpers/FileUtils.java | 81 --
.../ozone/container/common/helpers/KeyData.java | 170 ---
.../container/common/helpers/KeyUtils.java | 147 --
.../container/common/helpers/package-info.java | 22 -
.../container/common/impl/ChunkManagerImpl.java | 232 ----
.../impl/ContainerLocationManagerImpl.java | 149 --
.../common/impl/ContainerManagerImpl.java | 1101 ---------------
.../common/impl/ContainerReportManagerImpl.java | 87 --
.../container/common/impl/ContainerStatus.java | 217 ---
.../common/impl/ContainerStorageLocation.java | 203 ---
.../ozone/container/common/impl/Dispatcher.java | 708 ----------
.../container/common/impl/KeyManagerImpl.java | 201 ---
.../RandomContainerDeletionChoosingPolicy.java | 69 -
.../common/impl/StorageLocationReport.java | 63 -
...NOrderedContainerDeletionChoosingPolicy.java | 90 --
.../container/common/impl/package-info.java | 22 -
.../common/interfaces/ChunkManager.java | 75 -
.../ContainerDeletionChoosingPolicy.java | 45 -
.../common/interfaces/ContainerDispatcher.java | 49 -
.../interfaces/ContainerLocationManager.java | 58 -
.../ContainerLocationManagerMXBean.java | 36 -
.../common/interfaces/ContainerManager.java | 276 ----
.../interfaces/ContainerReportManager.java | 32 -
.../container/common/interfaces/KeyManager.java | 75 -
.../common/interfaces/package-info.java | 20 -
.../ozone/container/common/package-info.java | 28 -
.../statemachine/DatanodeStateMachine.java | 385 ------
.../statemachine/EndpointStateMachine.java | 293 ----
.../statemachine/EndpointStateMachineMBean.java | 34 -
.../statemachine/SCMConnectionManager.java | 201 ---
.../SCMConnectionManagerMXBean.java | 27 -
.../common/statemachine/StateContext.java | 281 ----
.../background/BlockDeletingService.java | 237 ----
.../statemachine/background/package-info.java | 18 -
.../commandhandler/CloseContainerHandler.java | 112 --
.../commandhandler/CommandDispatcher.java | 177 ---
.../commandhandler/CommandHandler.java | 59 -
.../commandhandler/ContainerReportHandler.java | 110 --
.../DeleteBlocksCommandHandler.java | 204 ---
.../commandhandler/package-info.java | 18 -
.../common/statemachine/package-info.java | 28 -
.../container/common/states/DatanodeState.java | 55 -
.../states/datanode/InitDatanodeState.java | 165 ---
.../states/datanode/RunningDatanodeState.java | 253 ----
.../common/states/datanode/package-info.java | 21 -
.../states/endpoint/HeartbeatEndpointTask.java | 270 ----
.../states/endpoint/RegisterEndpointTask.java | 198 ---
.../states/endpoint/VersionEndpointTask.java | 66 -
.../common/states/endpoint/package-info.java | 20 -
.../container/common/states/package-info.java | 18 -
.../common/transport/server/XceiverServer.java | 128 --
.../transport/server/XceiverServerHandler.java | 80 --
.../server/XceiverServerInitializer.java | 61 -
.../transport/server/XceiverServerSpi.java | 43 -
.../common/transport/server/package-info.java | 24 -
.../server/ratis/ContainerStateMachine.java | 290 ----
.../server/ratis/XceiverServerRatis.java | 214 ---
.../transport/server/ratis/package-info.java | 23 -
.../container/common/utils/ContainerCache.java | 168 ---
.../container/common/utils/package-info.java | 18 -
.../container/ozoneimpl/OzoneContainer.java | 269 ----
.../ozone/container/ozoneimpl/package-info.java | 21 -
.../apache/hadoop/ozone/ksm/BucketManager.java | 79 --
.../hadoop/ozone/ksm/BucketManagerImpl.java | 315 -----
.../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 -
.../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ----
.../ozone/ksm/KSMMetadataManagerImpl.java | 524 -------
.../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 437 ------
.../org/apache/hadoop/ozone/ksm/KSMStorage.java | 89 --
.../hadoop/ozone/ksm/KeyDeletingService.java | 141 --
.../org/apache/hadoop/ozone/ksm/KeyManager.java | 165 ---
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 512 -------
.../hadoop/ozone/ksm/KeySpaceManager.java | 902 ------------
.../ozone/ksm/KeySpaceManagerHttpServer.java | 78 --
.../hadoop/ozone/ksm/OpenKeyCleanupService.java | 116 --
.../ozone/ksm/ServiceListJSONServlet.java | 103 --
.../apache/hadoop/ozone/ksm/VolumeManager.java | 100 --
.../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ------
.../ozone/ksm/exceptions/KSMException.java | 116 --
.../ozone/ksm/exceptions/package-info.java | 19 -
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../org/apache/hadoop/ozone/lease/Lease.java | 189 ---
.../ozone/lease/LeaseAlreadyExistException.java | 46 -
.../ozone/lease/LeaseCallbackExecutor.java | 65 -
.../hadoop/ozone/lease/LeaseException.java | 45 -
.../ozone/lease/LeaseExpiredException.java | 45 -
.../apache/hadoop/ozone/lease/LeaseManager.java | 247 ----
.../lease/LeaseManagerNotRunningException.java | 45 -
.../ozone/lease/LeaseNotFoundException.java | 46 -
.../apache/hadoop/ozone/lease/package-info.java | 26 -
.../org/apache/hadoop/ozone/package-info.java | 35 -
.../StorageContainerDatanodeProtocol.java | 84 --
.../protocol/StorageContainerNodeProtocol.java | 69 -
.../hadoop/ozone/protocol/VersionResponse.java | 150 --
.../commands/CloseContainerCommand.java | 75 -
.../protocol/commands/DeleteBlocksCommand.java | 63 -
.../protocol/commands/RegisteredCommand.java | 177 ---
.../protocol/commands/ReregisterCommand.java | 59 -
.../ozone/protocol/commands/SCMCommand.java | 41 -
.../protocol/commands/SendContainerCommand.java | 80 --
.../ozone/protocol/commands/package-info.java | 21 -
.../hadoop/ozone/protocol/package-info.java | 23 -
...ceManagerProtocolServerSideTranslatorPB.java | 539 --------
...kLocationProtocolServerSideTranslatorPB.java | 169 ---
...rDatanodeProtocolClientSideTranslatorPB.java | 195 ---
.../StorageContainerDatanodeProtocolPB.java | 32 -
...rDatanodeProtocolServerSideTranslatorPB.java | 113 --
...rLocationProtocolServerSideTranslatorPB.java | 202 ---
.../hadoop/ozone/protocolPB/package-info.java | 24 -
.../org/apache/hadoop/ozone/scm/SCMMXBean.java | 50 -
.../org/apache/hadoop/ozone/scm/SCMStorage.java | 74 -
.../ozone/scm/StorageContainerManager.java | 1260 -----------------
.../scm/StorageContainerManagerHttpServer.java | 77 --
.../apache/hadoop/ozone/scm/VersionInfo.java | 81 --
.../hadoop/ozone/scm/block/BlockManager.java | 86 --
.../ozone/scm/block/BlockManagerImpl.java | 530 -------
.../ozone/scm/block/BlockmanagerMXBean.java | 30 -
.../block/DatanodeDeletedBlockTransactions.java | 130 --
.../hadoop/ozone/scm/block/DeletedBlockLog.java | 124 --
.../ozone/scm/block/DeletedBlockLogImpl.java | 351 -----
.../scm/block/SCMBlockDeletingService.java | 172 ---
.../hadoop/ozone/scm/block/package-info.java | 22 -
.../hadoop/ozone/scm/cli/OzoneBaseCLI.java | 44 -
.../ozone/scm/cli/OzoneCommandHandler.java | 79 --
.../apache/hadoop/ozone/scm/cli/ResultCode.java | 31 -
.../org/apache/hadoop/ozone/scm/cli/SCMCLI.java | 231 ----
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 720 ----------
.../cli/container/CloseContainerHandler.java | 84 --
.../cli/container/ContainerCommandHandler.java | 128 --
.../cli/container/CreateContainerHandler.java | 81 --
.../cli/container/DeleteContainerHandler.java | 92 --
.../scm/cli/container/InfoContainerHandler.java | 115 --
.../scm/cli/container/ListContainerHandler.java | 121 --
.../ozone/scm/cli/container/package-info.java | 19 -
.../hadoop/ozone/scm/cli/package-info.java | 19 -
.../ozone/scm/container/ContainerMapping.java | 606 --------
.../scm/container/ContainerStateManager.java | 451 ------
.../ContainerStates/ContainerAttribute.java | 244 ----
.../ContainerStates/ContainerState.java | 96 --
.../ContainerStates/ContainerStateMap.java | 402 ------
.../container/ContainerStates/package-info.java | 22 -
.../hadoop/ozone/scm/container/Mapping.java | 107 --
.../scm/container/closer/ContainerCloser.java | 192 ---
.../scm/container/closer/package-info.java | 23 -
.../ozone/scm/container/package-info.java | 22 -
.../algorithms/ContainerPlacementPolicy.java | 41 -
.../placement/algorithms/SCMCommonPolicy.java | 194 ---
.../SCMContainerPlacementCapacity.java | 133 --
.../algorithms/SCMContainerPlacementRandom.java | 89 --
.../placement/algorithms/package-info.java | 18 -
.../placement/metrics/ContainerStat.java | 166 ---
.../placement/metrics/DatanodeMetric.java | 91 --
.../container/placement/metrics/LongMetric.java | 162 ---
.../container/placement/metrics/NodeStat.java | 67 -
.../container/placement/metrics/SCMMetrics.java | 154 ---
.../placement/metrics/SCMNodeMetric.java | 223 ---
.../placement/metrics/SCMNodeStat.java | 139 --
.../placement/metrics/package-info.java | 20 -
.../scm/container/placement/package-info.java | 19 -
.../replication/ContainerSupervisor.java | 333 -----
.../container/replication/InProgressPool.java | 313 -----
.../scm/container/replication/PeriodicPool.java | 119 --
.../scm/container/replication/package-info.java | 23 -
.../ozone/scm/exceptions/SCMException.java | 119 --
.../ozone/scm/exceptions/package-info.java | 18 -
.../hadoop/ozone/scm/node/CommandQueue.java | 190 ---
.../ozone/scm/node/HeartbeatQueueItem.java | 116 --
.../hadoop/ozone/scm/node/NodeManager.java | 152 --
.../ozone/scm/node/NodeManagerMXBean.java | 57 -
.../hadoop/ozone/scm/node/NodePoolManager.java | 72 -
.../hadoop/ozone/scm/node/SCMNodeManager.java | 883 ------------
.../ozone/scm/node/SCMNodePoolManager.java | 266 ----
.../hadoop/ozone/scm/node/package-info.java | 31 -
.../apache/hadoop/ozone/scm/package-info.java | 22 -
.../ozone/scm/pipelines/PipelineManager.java | 176 ---
.../ozone/scm/pipelines/PipelineSelector.java | 227 ---
.../ozone/scm/pipelines/package-info.java | 38 -
.../scm/pipelines/ratis/RatisManagerImpl.java | 155 ---
.../ozone/scm/pipelines/ratis/package-info.java | 18 -
.../standalone/StandaloneManagerImpl.java | 142 --
.../scm/pipelines/standalone/package-info.java | 18 -
.../hadoop/ozone/scm/ratis/package-info.java | 22 -
.../org/apache/hadoop/ozone/tools/Freon.java | 1146 ---------------
.../apache/hadoop/ozone/tools/package-info.java | 22 -
.../ozone/web/ObjectStoreApplication.java | 59 -
.../hadoop/ozone/web/OzoneHttpServer.java | 216 ---
.../hadoop/ozone/web/client/OzoneBucket.java | 642 ---------
.../hadoop/ozone/web/client/OzoneKey.java | 44 -
.../ozone/web/client/OzoneRestClient.java | 801 -----------
.../web/client/OzoneRestClientException.java | 54 -
.../hadoop/ozone/web/client/OzoneVolume.java | 581 --------
.../hadoop/ozone/web/client/package-info.java | 34 -
.../hadoop/ozone/web/exceptions/ErrorTable.java | 225 ---
.../ozone/web/exceptions/package-info.java | 22 -
.../hadoop/ozone/web/handlers/BucketArgs.java | 214 ---
.../ozone/web/handlers/BucketHandler.java | 197 ---
.../web/handlers/BucketProcessTemplate.java | 323 -----
.../hadoop/ozone/web/handlers/KeyArgs.java | 117 --
.../hadoop/ozone/web/handlers/KeyHandler.java | 245 ----
.../ozone/web/handlers/KeyProcessTemplate.java | 233 ----
.../hadoop/ozone/web/handlers/ListArgs.java | 142 --
.../ozone/web/handlers/ServiceFilter.java | 61 -
.../web/handlers/StorageHandlerBuilder.java | 76 -
.../hadoop/ozone/web/handlers/UserArgs.java | 168 ---
.../ozone/web/handlers/UserHandlerBuilder.java | 75 -
.../hadoop/ozone/web/handlers/VolumeArgs.java | 142 --
.../ozone/web/handlers/VolumeHandler.java | 272 ----
.../web/handlers/VolumeProcessTemplate.java | 274 ----
.../hadoop/ozone/web/handlers/package-info.java | 22 -
.../hadoop/ozone/web/interfaces/Accounting.java | 57 -
.../hadoop/ozone/web/interfaces/Bucket.java | 183 ---
.../hadoop/ozone/web/interfaces/Keys.java | 146 --
.../ozone/web/interfaces/StorageHandler.java | 295 ----
.../hadoop/ozone/web/interfaces/UserAuth.java | 101 --
.../hadoop/ozone/web/interfaces/Volume.java | 182 ---
.../ozone/web/interfaces/package-info.java | 22 -
.../web/localstorage/LocalStorageHandler.java | 379 -----
.../web/localstorage/OzoneMetadataManager.java | 1138 ---------------
.../ozone/web/localstorage/package-info.java | 18 -
.../LengthInputStreamMessageBodyWriter.java | 59 -
.../web/messages/StringMessageBodyWriter.java | 62 -
.../hadoop/ozone/web/messages/package-info.java | 18 -
.../web/netty/CloseableCleanupListener.java | 46 -
.../web/netty/ObjectStoreChannelHandler.java | 78 --
.../web/netty/ObjectStoreJerseyContainer.java | 348 -----
.../ObjectStoreJerseyContainerProvider.java | 40 -
...RequestContentObjectStoreChannelHandler.java | 117 --
...equestDispatchObjectStoreChannelHandler.java | 103 --
.../hadoop/ozone/web/netty/package-info.java | 26 -
.../hadoop/ozone/web/ozShell/Handler.java | 76 -
.../apache/hadoop/ozone/web/ozShell/Shell.java | 415 ------
.../web/ozShell/bucket/CreateBucketHandler.java | 96 --
.../web/ozShell/bucket/DeleteBucketHandler.java | 87 --
.../web/ozShell/bucket/InfoBucketHandler.java | 94 --
.../web/ozShell/bucket/ListBucketHandler.java | 113 --
.../web/ozShell/bucket/UpdateBucketHandler.java | 94 --
.../ozone/web/ozShell/bucket/package-info.java | 23 -
.../web/ozShell/keys/DeleteKeyHandler.java | 96 --
.../ozone/web/ozShell/keys/GetKeyHandler.java | 116 --
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 98 --
.../ozone/web/ozShell/keys/ListKeyHandler.java | 116 --
.../ozone/web/ozShell/keys/PutKeyHandler.java | 106 --
.../ozone/web/ozShell/keys/package-info.java | 23 -
.../hadoop/ozone/web/ozShell/package-info.java | 27 -
.../web/ozShell/volume/CreateVolumeHandler.java | 99 --
.../web/ozShell/volume/DeleteVolumeHandler.java | 81 --
.../web/ozShell/volume/InfoVolumeHandler.java | 95 --
.../web/ozShell/volume/ListVolumeHandler.java | 114 --
.../web/ozShell/volume/UpdateVolumeHandler.java | 92 --
.../ozone/web/ozShell/volume/package-info.java | 23 -
.../apache/hadoop/ozone/web/package-info.java | 24 -
.../hadoop/ozone/web/request/OzoneQuota.java | 214 ---
.../hadoop/ozone/web/request/package-info.java | 23 -
.../hadoop/ozone/web/response/BucketInfo.java | 325 -----
.../hadoop/ozone/web/response/KeyInfo.java | 290 ----
.../hadoop/ozone/web/response/ListBuckets.java | 154 ---
.../hadoop/ozone/web/response/ListKeys.java | 209 ---
.../hadoop/ozone/web/response/ListVolumes.java | 152 --
.../hadoop/ozone/web/response/VolumeInfo.java | 308 -----
.../hadoop/ozone/web/response/VolumeOwner.java | 60 -
.../hadoop/ozone/web/response/package-info.java | 23 -
.../web/storage/DistributedStorageHandler.java | 539 --------
.../hadoop/ozone/web/storage/package-info.java | 27 -
.../hadoop/ozone/web/userauth/Simple.java | 169 ---
.../hadoop/ozone/web/userauth/package-info.java | 23 -
.../hadoop/ozone/web/utils/JsonUtils.java | 71 -
.../hadoop/ozone/web/utils/OzoneUtils.java | 303 ----
.../hadoop/ozone/web/utils/package-info.java | 18 -
.../apache/hadoop/utils/BackgroundService.java | 161 ---
.../org/apache/hadoop/utils/BackgroundTask.java | 28 -
.../hadoop/utils/BackgroundTaskQueue.java | 64 -
.../hadoop/utils/BackgroundTaskResult.java | 44 -
.../org/apache/hadoop/utils/BatchOperation.java | 90 --
.../org/apache/hadoop/utils/EntryConsumer.java | 38 -
.../org/apache/hadoop/utils/LevelDBStore.java | 379 -----
.../apache/hadoop/utils/MetadataKeyFilters.java | 118 --
.../org/apache/hadoop/utils/MetadataStore.java | 172 ---
.../hadoop/utils/MetadataStoreBuilder.java | 125 --
.../org/apache/hadoop/utils/RocksDBStore.java | 382 -----
.../apache/hadoop/utils/RocksDBStoreMBean.java | 148 --
.../org/apache/hadoop/utils/package-info.java | 18 -
.../main/proto/CBlockClientServerProtocol.proto | 93 --
.../src/main/proto/CBlockServiceProtocol.proto | 133 --
.../StorageContainerDatanodeProtocol.proto | 367 -----
...m.sun.jersey.spi.container.ContainerProvider | 16 -
.../src/main/resources/ozone-default.xml | 1308 ------------------
.../hadoop-hdfs/src/main/webapps/datanode/dn.js | 28 +-
.../hadoop-hdfs/src/main/webapps/ksm/index.html | 70 -
.../src/main/webapps/ksm/ksm-metrics.html | 44 -
.../hadoop-hdfs/src/main/webapps/ksm/ksm.js | 108 --
.../hadoop-hdfs/src/main/webapps/ksm/main.css | 23 -
.../hadoop-hdfs/src/main/webapps/ksm/main.html | 18 -
.../hadoop-hdfs/src/main/webapps/scm/index.html | 76 -
.../hadoop-hdfs/src/main/webapps/scm/main.html | 20 -
.../src/main/webapps/scm/scm-overview.html | 60 -
.../hadoop-hdfs/src/main/webapps/scm/scm.js | 54 -
.../main/webapps/static/angular-1.6.4.min.js | 332 -----
.../webapps/static/angular-nvd3-1.0.9.min.js | 1 -
.../webapps/static/angular-route-1.6.4.min.js | 17 -
.../src/main/webapps/static/d3-3.5.17.min.js | 5 -
.../src/main/webapps/static/dfs-dust.js | 5 +-
.../src/main/webapps/static/nvd3-1.8.5.min.css | 2 -
.../main/webapps/static/nvd3-1.8.5.min.css.map | 1 -
.../src/main/webapps/static/nvd3-1.8.5.min.js | 11 -
.../main/webapps/static/nvd3-1.8.5.min.js.map | 1 -
.../src/main/webapps/static/ozone.css | 60 -
.../src/main/webapps/static/ozone.js | 355 -----
.../main/webapps/static/templates/config.html | 92 --
.../src/main/webapps/static/templates/jvm.html | 26 -
.../src/main/webapps/static/templates/menu.html | 60 -
.../main/webapps/static/templates/overview.html | 39 -
.../webapps/static/templates/rpc-metrics.html | 87 --
.../src/site/markdown/OzoneCommandShell.md | 150 --
.../src/site/markdown/OzoneGettingStarted.md.vm | 320 -----
.../src/site/markdown/OzoneMetrics.md | 166 ---
.../src/site/markdown/OzoneOverview.md | 88 --
.../hadoop-hdfs/src/site/markdown/OzoneRest.md | 549 --------
.../apache/hadoop/cblock/TestBufferManager.java | 456 ------
.../org/apache/hadoop/cblock/TestCBlockCLI.java | 242 ----
.../hadoop/cblock/TestCBlockReadWrite.java | 377 -----
.../apache/hadoop/cblock/TestCBlockServer.java | 212 ---
.../cblock/TestCBlockServerPersistence.java | 132 --
.../hadoop/cblock/TestLocalBlockCache.java | 444 ------
.../kubernetes/TestDynamicProvisioner.java | 73 -
.../cblock/util/ContainerLookUpService.java | 73 -
.../hadoop/cblock/util/MockStorageClient.java | 176 ---
.../server/namenode/TestNameNodeHttpServer.java | 9 +-
.../namenode/ha/TestPipelinesFailover.java | 6 +-
.../hadoop/ozone/MiniOzoneClassicCluster.java | 606 --------
.../apache/hadoop/ozone/MiniOzoneCluster.java | 62 -
.../apache/hadoop/ozone/RatisTestHelper.java | 111 --
.../hadoop/ozone/TestContainerOperations.java | 90 --
.../apache/hadoop/ozone/TestMetadataStore.java | 414 ------
.../hadoop/ozone/TestMiniOzoneCluster.java | 254 ----
.../ozone/TestOzoneConfigurationFields.java | 39 -
.../ozone/TestStorageContainerManager.java | 447 ------
.../TestStorageContainerManagerHelper.java | 180 ---
.../ozone/client/TestOzoneClientUtils.java | 362 -----
.../hadoop/ozone/client/package-info.java | 23 -
.../ozone/client/rest/TestOzoneRestClient.java | 405 ------
.../hadoop/ozone/client/rest/package-info.java | 23 -
.../ozone/client/rpc/TestOzoneRpcClient.java | 815 -----------
.../hadoop/ozone/client/rpc/package-info.java | 23 -
.../ozone/container/ContainerTestHelper.java | 595 --------
.../TestUtils/BlockDeletingServiceTestImpl.java | 103 --
.../ReplicationDatanodeStateManager.java | 97 --
.../TestUtils/ReplicationNodeManagerMock.java | 323 -----
.../ReplicationNodePoolManagerMock.java | 132 --
.../ozone/container/TestUtils/package-info.java | 18 -
.../ozone/container/common/SCMTestUtils.java | 221 ---
.../ozone/container/common/ScmTestMock.java | 261 ----
.../common/TestBlockDeletingService.java | 396 ------
.../common/TestDatanodeStateMachine.java | 361 -----
.../ozone/container/common/TestEndPoint.java | 461 ------
.../TestContainerDeletionChoosingPolicy.java | 197 ---
.../common/impl/TestContainerPersistence.java | 905 ------------
.../TestCloseContainerHandler.java | 112 --
.../container/metrics/TestContainerMetrics.java | 124 --
.../container/ozoneimpl/TestOzoneContainer.java | 584 --------
.../ozoneimpl/TestOzoneContainerRatis.java | 143 --
.../container/ozoneimpl/TestRatisManager.java | 129 --
.../placement/TestContainerPlacement.java | 131 --
.../placement/TestDatanodeMetrics.java | 59 -
.../replication/TestContainerSupervisor.java | 269 ----
.../container/replication/package-info.java | 18 -
.../transport/server/TestContainerServer.java | 258 ----
.../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ------
.../hadoop/ozone/ksm/TestChunkStreams.java | 234 ----
.../ozone/ksm/TestContainerReportWithKeys.java | 138 --
.../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 307 ----
.../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 273 ----
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 1238 -----------------
.../ksm/TestKeySpaceManagerHttpServer.java | 141 --
.../ksm/TestKeySpaceManagerRestInterface.java | 141 --
.../ozone/ksm/TestKsmBlockVersioning.java | 254 ----
.../ksm/TestMultipleContainerReadWrite.java | 216 ---
.../apache/hadoop/ozone/ksm/package-info.java | 21 -
.../hadoop/ozone/lease/TestLeaseManager.java | 374 -----
.../apache/hadoop/ozone/lease/package-info.java | 21 -
.../hadoop/ozone/ozShell/TestOzoneShell.java | 804 -----------
.../org/apache/hadoop/ozone/package-info.java | 21 -
.../hadoop/ozone/scm/TestAllocateContainer.java | 104 --
.../hadoop/ozone/scm/TestContainerSQLCli.java | 300 ----
.../ozone/scm/TestContainerSmallFile.java | 151 --
.../org/apache/hadoop/ozone/scm/TestSCMCli.java | 535 -------
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 162 ---
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 251 ----
.../TestStorageContainerManagerHttpServer.java | 141 --
.../ozone/scm/TestXceiverClientManager.java | 196 ---
.../ozone/scm/TestXceiverClientMetrics.java | 172 ---
.../ozone/scm/block/TestBlockManager.java | 160 ---
.../ozone/scm/block/TestDeletedBlockLog.java | 348 -----
.../BenchmarkContainerStateMap.java | 180 ---
.../ContainerStates/TestContainerAttribute.java | 139 --
.../ContainerStates/TestContainerStateMap.java | 226 ---
.../ozone/scm/container/MockNodeManager.java | 519 -------
.../scm/container/TestContainerMapping.java | 333 -----
.../container/TestContainerStateManager.java | 303 ----
.../container/closer/TestContainerCloser.java | 221 ---
.../ozone/scm/node/TestContainerPlacement.java | 172 ---
.../hadoop/ozone/scm/node/TestNodeManager.java | 1145 ---------------
.../hadoop/ozone/scm/node/TestQueryNode.java | 122 --
.../ozone/scm/node/TestSCMNodePoolManager.java | 156 ---
.../apache/hadoop/ozone/scm/package-info.java | 21 -
.../hadoop/ozone/tools/TestDataValidate.java | 146 --
.../apache/hadoop/ozone/tools/TestFreon.java | 136 --
.../apache/hadoop/ozone/tools/package-info.java | 21 -
.../apache/hadoop/ozone/web/TestBucketInfo.java | 81 --
.../ozone/web/TestDistributedOzoneVolumes.java | 186 ---
.../apache/hadoop/ozone/web/TestErrorCode.java | 53 -
.../hadoop/ozone/web/TestLocalOzoneVolumes.java | 185 ---
.../hadoop/ozone/web/TestOzoneHelper.java | 414 ------
.../hadoop/ozone/web/TestOzoneHttpServer.java | 98 --
.../ozone/web/TestOzoneRestWithMiniCluster.java | 258 ----
.../hadoop/ozone/web/TestOzoneWebAccess.java | 126 --
.../org/apache/hadoop/ozone/web/TestQuota.java | 116 --
.../org/apache/hadoop/ozone/web/TestUtils.java | 101 --
.../hadoop/ozone/web/TestVolumeStructs.java | 73 -
.../hadoop/ozone/web/client/TestBuckets.java | 251 ----
.../ozone/web/client/TestBucketsRatis.java | 76 -
.../hadoop/ozone/web/client/TestKeys.java | 688 ---------
.../hadoop/ozone/web/client/TestKeysRatis.java | 126 --
.../ozone/web/client/TestOzoneClient.java | 305 ----
.../hadoop/ozone/web/client/TestVolume.java | 434 ------
.../ozone/web/client/TestVolumeRatis.java | 131 --
.../apache/hadoop/ozone/web/package-info.java | 21 -
.../org/apache/hadoop/test/OzoneTestDriver.java | 59 -
.../hadoop/utils/TestRocksDBStoreMBean.java | 87 --
.../dynamicprovisioner/expected1-pv.json | 37 -
.../dynamicprovisioner/input1-pvc.json | 38 -
.../src/test/resources/ozone-site.xml | 24 -
hadoop-hdsl/client/pom.xml | 48 +
.../hadoop/ozone/client/OzoneClientUtils.java | 231 ++++
.../hadoop/ozone/client/package-info.java | 23 +
.../org/apache/hadoop/scm/XceiverClient.java | 192 +++
.../apache/hadoop/scm/XceiverClientHandler.java | 204 +++
.../hadoop/scm/XceiverClientInitializer.java | 72 +
.../apache/hadoop/scm/XceiverClientManager.java | 219 +++
.../apache/hadoop/scm/XceiverClientMetrics.java | 92 ++
.../apache/hadoop/scm/XceiverClientRatis.java | 264 ++++
.../scm/client/ContainerOperationClient.java | 403 ++++++
.../apache/hadoop/scm/client/package-info.java | 23 +
.../org/apache/hadoop/scm/package-info.java | 23 +
.../hadoop/scm/storage/ChunkInputStream.java | 261 ++++
.../hadoop/scm/storage/ChunkOutputStream.java | 227 +++
.../apache/hadoop/scm/storage/package-info.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 21 +
hadoop-hdsl/common/pom.xml | 146 ++
.../java/org/apache/hadoop/hdsl/HdslUtils.java | 273 ++++
.../hadoop/hdsl/conf/OzoneConfiguration.java | 161 +++
.../apache/hadoop/hdsl/conf/package-info.java | 18 +
.../org/apache/hadoop/hdsl/package-info.java | 23 +
.../java/org/apache/hadoop/ozone/OzoneAcl.java | 231 ++++
.../apache/hadoop/ozone/OzoneConfigKeys.java | 239 ++++
.../org/apache/hadoop/ozone/OzoneConsts.java | 167 +++
.../apache/hadoop/ozone/client/OzoneQuota.java | 203 +++
.../hadoop/ozone/client/ReplicationFactor.java | 63 +
.../hadoop/ozone/client/ReplicationType.java | 28 +
.../hadoop/ozone/client/package-info.java | 23 +
.../apache/hadoop/ozone/common/BlockGroup.java | 87 ++
.../ozone/common/DeleteBlockGroupResult.java | 94 ++
.../InconsistentStorageStateException.java | 51 +
.../org/apache/hadoop/ozone/common/Storage.java | 249 ++++
.../apache/hadoop/ozone/common/StorageInfo.java | 184 +++
.../hadoop/ozone/common/package-info.java | 18 +
.../InvalidStateTransitionException.java | 42 +
.../ozone/common/statemachine/StateMachine.java | 68 +
.../ozone/common/statemachine/package-info.java | 21 +
.../container/common/helpers/ChunkInfo.java | 185 +++
.../ozone/container/common/helpers/KeyData.java | 170 +++
.../container/common/helpers/package-info.java | 23 +
.../org/apache/hadoop/ozone/lease/Lease.java | 189 +++
.../ozone/lease/LeaseAlreadyExistException.java | 46 +
.../ozone/lease/LeaseCallbackExecutor.java | 65 +
.../hadoop/ozone/lease/LeaseException.java | 45 +
.../ozone/lease/LeaseExpiredException.java | 45 +
.../apache/hadoop/ozone/lease/LeaseManager.java | 247 ++++
.../lease/LeaseManagerNotRunningException.java | 45 +
.../ozone/lease/LeaseNotFoundException.java | 46 +
.../apache/hadoop/ozone/lease/package-info.java | 26 +
.../org/apache/hadoop/ozone/package-info.java | 35 +
...kLocationProtocolServerSideTranslatorPB.java | 169 +++
...rLocationProtocolServerSideTranslatorPB.java | 202 +++
.../hadoop/ozone/protocolPB/package-info.java | 24 +
.../hadoop/ozone/web/utils/JsonUtils.java | 71 +
.../org/apache/hadoop/scm/ScmConfigKeys.java | 271 ++++
.../java/org/apache/hadoop/scm/ScmInfo.java | 81 ++
.../org/apache/hadoop/scm/XceiverClientSpi.java | 129 ++
.../org/apache/hadoop/scm/client/ScmClient.java | 139 ++
.../apache/hadoop/scm/client/package-info.java | 24 +
.../container/ContainerStates/ContainerID.java | 97 ++
.../container/ContainerStates/package-info.java | 22 +
.../common/helpers/AllocatedBlock.java | 77 ++
.../container/common/helpers/ContainerInfo.java | 333 +++++
.../common/helpers/DeleteBlockResult.java | 52 +
.../scm/container/common/helpers/Pipeline.java | 253 ++++
.../common/helpers/PipelineChannel.java | 121 ++
.../helpers/StorageContainerException.java | 104 ++
.../container/common/helpers/package-info.java | 22 +
.../org/apache/hadoop/scm/package-info.java | 24 +
.../hadoop/scm/protocol/LocatedContainer.java | 127 ++
.../scm/protocol/ScmBlockLocationProtocol.java | 72 +
.../hadoop/scm/protocol/ScmLocatedBlock.java | 100 ++
.../StorageContainerLocationProtocol.java | 123 ++
.../hadoop/scm/protocol/package-info.java | 19 +
...kLocationProtocolClientSideTranslatorPB.java | 207 +++
.../protocolPB/ScmBlockLocationProtocolPB.java | 35 +
...rLocationProtocolClientSideTranslatorPB.java | 304 ++++
.../StorageContainerLocationProtocolPB.java | 34 +
.../hadoop/scm/protocolPB/package-info.java | 24 +
.../scm/storage/ContainerProtocolCalls.java | 397 ++++++
.../apache/hadoop/scm/storage/package-info.java | 23 +
.../apache/hadoop/utils/BackgroundService.java | 161 +++
.../org/apache/hadoop/utils/BackgroundTask.java | 28 +
.../hadoop/utils/BackgroundTaskQueue.java | 64 +
.../hadoop/utils/BackgroundTaskResult.java | 44 +
.../org/apache/hadoop/utils/BatchOperation.java | 90 ++
.../org/apache/hadoop/utils/EntryConsumer.java | 38 +
.../org/apache/hadoop/utils/LevelDBStore.java | 380 +++++
.../apache/hadoop/utils/MetadataKeyFilters.java | 118 ++
.../org/apache/hadoop/utils/MetadataStore.java | 172 +++
.../hadoop/utils/MetadataStoreBuilder.java | 125 ++
.../org/apache/hadoop/utils/RocksDBStore.java | 382 +++++
.../apache/hadoop/utils/RocksDBStoreMBean.java | 136 ++
.../org/apache/hadoop/utils/package-info.java | 18 +
.../main/java/org/apache/ratis/RatisHelper.java | 128 ++
.../java/org/apache/ratis/package-info.java | 22 +
.../com/google/protobuf/ShadedProtoUtil.java | 38 +
.../com/google/protobuf/package-info.java | 22 +
.../main/proto/DatanodeContainerProtocol.proto | 415 ++++++
.../main/proto/ScmBlockLocationProtocol.proto | 166 +++
.../StorageContainerLocationProtocol.proto | 214 +++
hadoop-hdsl/common/src/main/proto/hdsl.proto | 162 +++
.../common/src/main/resources/ozone-default.xml | 1031 ++++++++++++++
.../apache/hadoop/ozone/TestMetadataStore.java | 414 ++++++
.../org/apache/hadoop/ozone/TestOzoneAcls.java | 141 ++
.../hadoop/ozone/common/TestStateMachine.java | 104 ++
.../hadoop/ozone/lease/TestLeaseManager.java | 374 +++++
.../apache/hadoop/ozone/lease/package-info.java | 21 +
.../org/apache/hadoop/ozone/package-info.java | 21 +
.../java/org/apache/hadoop/scm/TestArchive.java | 114 ++
.../org/apache/hadoop/scm/package-info.java | 21 +
.../hadoop/utils/TestRocksDBStoreMBean.java | 87 ++
hadoop-hdsl/container-service/pom.xml | 114 ++
.../apache/hadoop/ozone/HdslServerPlugin.java | 82 ++
.../container/common/helpers/ChunkUtils.java | 345 +++++
.../container/common/helpers/ContainerData.java | 326 +++++
.../common/helpers/ContainerMetrics.java | 121 ++
.../common/helpers/ContainerReport.java | 218 +++
.../common/helpers/ContainerUtils.java | 477 +++++++
.../helpers/DeletedContainerBlocksSummary.java | 103 ++
.../container/common/helpers/FileUtils.java | 81 ++
.../container/common/helpers/KeyUtils.java | 147 ++
.../container/common/helpers/package-info.java | 22 +
.../container/common/impl/ChunkManagerImpl.java | 232 ++++
.../impl/ContainerLocationManagerImpl.java | 149 ++
.../common/impl/ContainerManagerImpl.java | 1101 +++++++++++++++
.../common/impl/ContainerReportManagerImpl.java | 89 ++
.../container/common/impl/ContainerStatus.java | 217 +++
.../common/impl/ContainerStorageLocation.java | 203 +++
.../ozone/container/common/impl/Dispatcher.java | 708 ++++++++++
.../container/common/impl/KeyManagerImpl.java | 201 +++
.../RandomContainerDeletionChoosingPolicy.java | 69 +
.../common/impl/StorageLocationReport.java | 63 +
...NOrderedContainerDeletionChoosingPolicy.java | 90 ++
.../container/common/impl/package-info.java | 22 +
.../common/interfaces/ChunkManager.java | 75 +
.../ContainerDeletionChoosingPolicy.java | 45 +
.../common/interfaces/ContainerDispatcher.java | 49 +
.../interfaces/ContainerLocationManager.java | 58 +
.../ContainerLocationManagerMXBean.java | 36 +
.../common/interfaces/ContainerManager.java | 277 ++++
.../interfaces/ContainerReportManager.java | 32 +
.../container/common/interfaces/KeyManager.java | 75 +
.../common/interfaces/package-info.java | 20 +
.../ozone/container/common/package-info.java | 28 +
.../statemachine/DatanodeStateMachine.java | 386 ++++++
.../statemachine/EndpointStateMachine.java | 295 ++++
.../statemachine/EndpointStateMachineMBean.java | 34 +
.../statemachine/SCMConnectionManager.java | 203 +++
.../SCMConnectionManagerMXBean.java | 27 +
.../common/statemachine/StateContext.java | 281 ++++
.../background/BlockDeletingService.java | 238 ++++
.../statemachine/background/package-info.java | 18 +
.../commandhandler/CloseContainerHandler.java | 112 ++
.../commandhandler/CommandDispatcher.java | 177 +++
.../commandhandler/CommandHandler.java | 59 +
.../commandhandler/ContainerReportHandler.java | 110 ++
.../DeleteBlocksCommandHandler.java | 204 +++
.../commandhandler/package-info.java | 18 +
.../common/statemachine/package-info.java | 28 +
.../container/common/states/DatanodeState.java | 55 +
.../states/datanode/InitDatanodeState.java | 167 +++
.../states/datanode/RunningDatanodeState.java | 253 ++++
.../common/states/datanode/package-info.java | 21 +
.../states/endpoint/HeartbeatEndpointTask.java | 270 ++++
.../states/endpoint/RegisterEndpointTask.java | 198 +++
.../states/endpoint/VersionEndpointTask.java | 66 +
.../common/states/endpoint/package-info.java | 20 +
.../container/common/states/package-info.java | 18 +
.../common/transport/server/XceiverServer.java | 128 ++
.../transport/server/XceiverServerHandler.java | 80 ++
.../server/XceiverServerInitializer.java | 61 +
.../transport/server/XceiverServerSpi.java | 43 +
.../common/transport/server/package-info.java | 24 +
.../server/ratis/ContainerStateMachine.java | 290 ++++
.../server/ratis/XceiverServerRatis.java | 214 +++
.../transport/server/ratis/package-info.java | 23 +
.../container/common/utils/ContainerCache.java | 168 +++
.../container/common/utils/package-info.java | 18 +
.../container/ozoneimpl/OzoneContainer.java | 269 ++++
.../ozone/container/ozoneimpl/package-info.java | 21 +
.../org/apache/hadoop/ozone/package-info.java | 23 +
.../StorageContainerDatanodeProtocol.java | 84 ++
.../protocol/StorageContainerNodeProtocol.java | 69 +
.../hadoop/ozone/protocol/VersionResponse.java | 150 ++
.../commands/CloseContainerCommand.java | 75 +
.../protocol/commands/DeleteBlocksCommand.java | 63 +
.../protocol/commands/RegisteredCommand.java | 177 +++
.../protocol/commands/ReregisterCommand.java | 58 +
.../ozone/protocol/commands/SCMCommand.java | 41 +
.../protocol/commands/SendContainerCommand.java | 80 ++
.../ozone/protocol/commands/package-info.java | 21 +
.../hadoop/ozone/protocol/package-info.java | 23 +
...rDatanodeProtocolClientSideTranslatorPB.java | 195 +++
.../StorageContainerDatanodeProtocolPB.java | 32 +
...rDatanodeProtocolServerSideTranslatorPB.java | 113 ++
.../apache/hadoop/ozone/scm/HdslServerUtil.java | 304 ++++
.../apache/hadoop/ozone/scm/VersionInfo.java | 81 ++
.../StorageContainerDatanodeProtocol.proto | 367 +++++
...m.sun.jersey.spi.container.ContainerProvider | 16 +
.../container/common/ContainerTestUtils.java | 68 +
.../ozone/container/common/SCMTestUtils.java | 121 ++
.../ozone/container/common/ScmTestMock.java | 261 ++++
.../common/TestDatanodeStateMachine.java | 361 +++++
.../testutils/BlockDeletingServiceTestImpl.java | 103 ++
.../ozone/container/testutils/package-info.java | 18 +
hadoop-hdsl/framework/pom.xml | 102 ++
.../hadoop/ozone/web/OzoneHttpServer.java | 218 +++
.../apache/hadoop/ozone/web/package-info.java | 23 +
.../hadoop/ozone/web/util/ServerUtils.java | 140 ++
.../hadoop/ozone/web/util/package-info.java | 23 +
.../framework/src/main/webapps/datanode/dn.js | 92 ++
.../main/webapps/static/angular-1.6.4.min.js | 332 +++++
.../webapps/static/angular-nvd3-1.0.9.min.js | 1 +
.../webapps/static/angular-route-1.6.4.min.js | 17 +
.../src/main/webapps/static/d3-3.5.17.min.js | 5 +
.../src/main/webapps/static/dfs-dust.js | 133 ++
.../src/main/webapps/static/nvd3-1.8.5.min.css | 2 +
.../main/webapps/static/nvd3-1.8.5.min.css.map | 1 +
.../src/main/webapps/static/nvd3-1.8.5.min.js | 11 +
.../main/webapps/static/nvd3-1.8.5.min.js.map | 1 +
.../framework/src/main/webapps/static/ozone.css | 60 +
.../framework/src/main/webapps/static/ozone.js | 355 +++++
.../main/webapps/static/templates/config.html | 91 ++
.../src/main/webapps/static/templates/jvm.html | 26 +
.../src/main/webapps/static/templates/menu.html | 60 +
.../main/webapps/static/templates/overview.html | 39 +
.../webapps/static/templates/rpc-metrics.html | 87 ++
.../framework/src/test/resources/ozone-site.xml | 24 +
hadoop-hdsl/pom.xml | 60 +
hadoop-hdsl/server-scm/pom.xml | 141 ++
.../org/apache/hadoop/ozone/scm/SCMMXBean.java | 50 +
.../org/apache/hadoop/ozone/scm/SCMStorage.java | 73 +
.../ozone/scm/StorageContainerManager.java | 1269 +++++++++++++++++
.../scm/StorageContainerManagerHttpServer.java | 77 ++
.../hadoop/ozone/scm/block/BlockManager.java | 86 ++
.../ozone/scm/block/BlockManagerImpl.java | 531 +++++++
.../ozone/scm/block/BlockmanagerMXBean.java | 30 +
.../block/DatanodeDeletedBlockTransactions.java | 130 ++
.../hadoop/ozone/scm/block/DeletedBlockLog.java | 124 ++
.../ozone/scm/block/DeletedBlockLogImpl.java | 352 +++++
.../scm/block/SCMBlockDeletingService.java | 172 +++
.../hadoop/ozone/scm/block/package-info.java | 22 +
.../ozone/scm/container/ContainerMapping.java | 607 ++++++++
.../scm/container/ContainerStateManager.java | 451 ++++++
.../ContainerStates/ContainerAttribute.java | 244 ++++
.../ContainerStates/ContainerState.java | 96 ++
.../ContainerStates/ContainerStateMap.java | 402 ++++++
.../container/ContainerStates/package-info.java | 22 +
.../hadoop/ozone/scm/container/Mapping.java | 107 ++
.../scm/container/closer/ContainerCloser.java | 192 +++
.../scm/container/closer/package-info.java | 23 +
.../ozone/scm/container/package-info.java | 22 +
.../algorithms/ContainerPlacementPolicy.java | 41 +
.../placement/algorithms/SCMCommonPolicy.java | 194 +++
.../SCMContainerPlacementCapacity.java | 133 ++
.../algorithms/SCMContainerPlacementRandom.java | 89 ++
.../placement/algorithms/package-info.java | 18 +
.../placement/metrics/ContainerStat.java | 166 +++
.../placement/metrics/DatanodeMetric.java | 91 ++
.../container/placement/metrics/LongMetric.java | 162 +++
.../container/placement/metrics/NodeStat.java | 67 +
.../container/placement/metrics/SCMMetrics.java | 154 +++
.../placement/metrics/SCMNodeMetric.java | 223 +++
.../placement/metrics/SCMNodeStat.java | 139 ++
.../placement/metrics/package-info.java | 20 +
.../scm/container/placement/package-info.java | 19 +
.../replication/ContainerSupervisor.java | 333 +++++
.../container/replication/InProgressPool.java | 313 +++++
.../scm/container/replication/PeriodicPool.java | 119 ++
.../scm/container/replication/package-info.java | 23 +
.../ozone/scm/exceptions/SCMException.java | 119 ++
.../ozone/scm/exceptions/package-info.java | 18 +
.../hadoop/ozone/scm/node/CommandQueue.java | 190 +++
.../ozone/scm/node/HeartbeatQueueItem.java | 116 ++
.../hadoop/ozone/scm/node/NodeManager.java | 152 ++
.../ozone/scm/node/NodeManagerMXBean.java | 57 +
.../hadoop/ozone/scm/node/NodePoolManager.java | 72 +
.../hadoop/ozone/scm/node/SCMNodeManager.java | 891 ++++++++++++
.../ozone/scm/node/SCMNodePoolManager.java | 267 ++++
.../hadoop/ozone/scm/node/package-info.java | 31 +
.../apache/hadoop/ozone/scm/package-info.java | 22 +
.../ozone/scm/pipelines/PipelineManager.java | 176 +++
.../ozone/scm/pipelines/PipelineSelector.java | 227 +++
.../ozone/scm/pipelines/package-info.java | 38 +
.../scm/pipelines/ratis/RatisManagerImpl.java | 155 +++
.../ozone/scm/pipelines/ratis/package-info.java | 18 +
.../standalone/StandaloneManagerImpl.java | 142 ++
.../scm/pipelines/standalone/package-info.java | 18 +
.../hadoop/ozone/scm/ratis/package-info.java | 22 +
.../server-scm/src/main/webapps/scm/index.html | 76 +
.../server-scm/src/main/webapps/scm/main.html | 20 +
.../src/main/webapps/scm/scm-overview.html | 60 +
.../server-scm/src/main/webapps/scm/scm.js | 54 +
.../ozone/container/common/TestEndPoint.java | 465 +++++++
.../placement/TestContainerPlacement.java | 131 ++
.../placement/TestDatanodeMetrics.java | 59 +
.../replication/TestContainerSupervisor.java | 268 ++++
.../container/replication/package-info.java | 18 +
.../ReplicationDatanodeStateManager.java | 97 ++
.../testutils/ReplicationNodeManagerMock.java | 323 +++++
.../ReplicationNodePoolManagerMock.java | 132 ++
.../ozone/container/testutils/package-info.java | 18 +
.../hadoop/ozone/scm/HdslServerUtilTest.java | 315 +++++
.../TestStorageContainerManagerHttpServer.java | 141 ++
.../org/apache/hadoop/ozone/scm/TestUtils.java | 103 ++
.../ozone/scm/block/TestBlockManager.java | 160 +++
.../ozone/scm/block/TestDeletedBlockLog.java | 348 +++++
.../ContainerStates/TestContainerAttribute.java | 142 ++
.../ozone/scm/container/MockNodeManager.java | 520 +++++++
.../scm/container/TestContainerMapping.java | 336 +++++
.../container/closer/TestContainerCloser.java | 221 +++
.../ozone/scm/node/TestContainerPlacement.java | 173 +++
.../hadoop/ozone/scm/node/TestNodeManager.java | 1148 +++++++++++++++
.../ozone/scm/node/TestSCMNodePoolManager.java | 158 +++
.../apache/hadoop/ozone/scm/package-info.java | 21 +
hadoop-hdsl/tools/pom.xml | 61 +
.../hadoop/ozone/scm/cli/OzoneBaseCLI.java | 43 +
.../ozone/scm/cli/OzoneCommandHandler.java | 87 ++
.../apache/hadoop/ozone/scm/cli/ResultCode.java | 31 +
.../org/apache/hadoop/ozone/scm/cli/SCMCLI.java | 230 +++
.../cli/container/CloseContainerHandler.java | 84 ++
.../cli/container/ContainerCommandHandler.java | 128 ++
.../cli/container/CreateContainerHandler.java | 81 ++
.../cli/container/DeleteContainerHandler.java | 92 ++
.../scm/cli/container/InfoContainerHandler.java | 115 ++
.../scm/cli/container/ListContainerHandler.java | 121 ++
.../ozone/scm/cli/container/package-info.java | 19 +
.../hadoop/ozone/scm/cli/package-info.java | 19 +
hadoop-ozone/client/pom.xml | 62 +
.../apache/hadoop/ozone/client/BucketArgs.java | 123 ++
.../apache/hadoop/ozone/client/ObjectStore.java | 210 +++
.../apache/hadoop/ozone/client/OzoneBucket.java | 360 +++++
.../apache/hadoop/ozone/client/OzoneClient.java | 103 ++
.../hadoop/ozone/client/OzoneClientFactory.java | 307 ++++
.../client/OzoneClientInvocationHandler.java | 62 +
.../apache/hadoop/ozone/client/OzoneKey.java | 119 ++
.../apache/hadoop/ozone/client/OzoneVolume.java | 293 ++++
.../apache/hadoop/ozone/client/VolumeArgs.java | 128 ++
.../ozone/client/io/ChunkGroupInputStream.java | 319 +++++
.../ozone/client/io/ChunkGroupOutputStream.java | 492 +++++++
.../client/io/OzoneContainerTranslation.java | 51 +
.../ozone/client/io/OzoneInputStream.java | 60 +
.../ozone/client/io/OzoneOutputStream.java | 64 +
.../hadoop/ozone/client/io/package-info.java | 23 +
.../hadoop/ozone/client/package-info.java | 23 +
.../ozone/client/protocol/ClientProtocol.java | 323 +++++
.../ozone/client/protocol/package-info.java | 23 +
.../client/rest/DefaultRestServerSelector.java | 36 +
.../ozone/client/rest/OzoneExceptionMapper.java | 45 +
.../hadoop/ozone/client/rest/RestClient.java | 799 +++++++++++
.../ozone/client/rest/RestServerSelector.java | 40 +
.../client/rest/exceptions/package-info.java | 22 +
.../ozone/client/rest/headers/package-info.java | 22 +
.../hadoop/ozone/client/rest/package-info.java | 23 +
.../hadoop/ozone/client/rpc/RpcClient.java | 567 ++++++++
.../hadoop/ozone/client/rpc/package-info.java | 23 +
.../hadoop/ozone/web/client/OzoneBucket.java | 644 +++++++++
.../hadoop/ozone/web/client/OzoneKey.java | 44 +
.../ozone/web/client/OzoneRestClient.java | 803 +++++++++++
.../web/client/OzoneRestClientException.java | 54 +
.../hadoop/ozone/web/client/OzoneVolume.java | 583 ++++++++
.../hadoop/ozone/web/client/package-info.java | 34 +
.../ozone/client/TestOzoneClientUtils.java | 105 ++
.../hadoop/ozone/client/package-info.java | 23 +
.../common/dev-support/findbugsExcludeFile.xml | 5 +
hadoop-ozone/common/pom.xml | 108 ++
hadoop-ozone/common/src/main/bin/oz | 332 +++++
hadoop-ozone/common/src/main/bin/start-ozone.sh | 105 ++
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 99 ++
.../common/src/main/conf/ozone-site.xml | 24 +
.../java/org/apache/hadoop/ozone/KsmUtils.java | 87 ++
.../ozone/client/io/LengthInputStream.java | 49 +
.../hadoop/ozone/client/io/package-info.java | 22 +
.../ozone/client/rest/OzoneException.java | 267 ++++
.../ozone/client/rest/headers/Header.java | 71 +
.../ozone/client/rest/headers/package-info.java | 22 +
.../hadoop/ozone/client/rest/package-info.java | 22 +
.../ozone/client/rest/response/BucketInfo.java | 231 ++++
.../ozone/client/rest/response/KeyInfo.java | 216 +++
.../ozone/client/rest/response/VolumeInfo.java | 215 +++
.../ozone/client/rest/response/VolumeOwner.java | 61 +
.../client/rest/response/package-info.java | 24 +
.../apache/hadoop/ozone/ksm/KSMConfigKeys.java | 81 ++
.../hadoop/ozone/ksm/helpers/KsmBucketArgs.java | 233 ++++
.../hadoop/ozone/ksm/helpers/KsmBucketInfo.java | 235 ++++
.../hadoop/ozone/ksm/helpers/KsmKeyArgs.java | 119 ++
.../hadoop/ozone/ksm/helpers/KsmKeyInfo.java | 243 ++++
.../ozone/ksm/helpers/KsmKeyLocationInfo.java | 136 ++
.../ksm/helpers/KsmKeyLocationInfoGroup.java | 118 ++
.../ozone/ksm/helpers/KsmOzoneAclMap.java | 110 ++
.../hadoop/ozone/ksm/helpers/KsmVolumeArgs.java | 223 +++
.../ozone/ksm/helpers/OpenKeySession.java | 50 +
.../hadoop/ozone/ksm/helpers/ServiceInfo.java | 237 ++++
.../hadoop/ozone/ksm/helpers/VolumeArgs.java | 140 ++
.../hadoop/ozone/ksm/helpers/package-info.java | 18 +
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
.../ksm/protocol/KeySpaceManagerProtocol.java | 245 ++++
.../hadoop/ozone/ksm/protocol/package-info.java | 19 +
...ceManagerProtocolClientSideTranslatorPB.java | 744 ++++++++++
.../protocolPB/KeySpaceManagerProtocolPB.java | 34 +
.../ozone/ksm/protocolPB/package-info.java | 19 +
.../org/apache/hadoop/ozone/package-info.java | 22 +
.../hadoop/ozone/protocolPB/KSMPBHelper.java | 113 ++
.../hadoop/ozone/protocolPB/package-info.java | 24 +
.../apache/hadoop/ozone/tools/OzoneGetConf.java | 269 ++++
.../apache/hadoop/ozone/tools/package-info.java | 21 +
.../hadoop/ozone/web/handlers/BucketArgs.java | 214 +++
.../hadoop/ozone/web/handlers/KeyArgs.java | 117 ++
.../hadoop/ozone/web/handlers/ListArgs.java | 142 ++
.../hadoop/ozone/web/handlers/UserArgs.java | 168 +++
.../hadoop/ozone/web/handlers/VolumeArgs.java | 142 ++
.../hadoop/ozone/web/handlers/package-info.java | 22 +
.../hadoop/ozone/web/request/OzoneQuota.java | 214 +++
.../hadoop/ozone/web/request/package-info.java | 23 +
.../hadoop/ozone/web/response/BucketInfo.java | 325 +++++
.../hadoop/ozone/web/response/KeyInfo.java | 290 ++++
.../hadoop/ozone/web/response/ListBuckets.java | 154 +++
.../hadoop/ozone/web/response/ListKeys.java | 209 +++
.../hadoop/ozone/web/response/ListVolumes.java | 152 ++
.../hadoop/ozone/web/response/VolumeInfo.java | 308 +++++
.../hadoop/ozone/web/response/VolumeOwner.java | 60 +
.../hadoop/ozone/web/response/package-info.java | 23 +
.../hadoop/ozone/web/utils/OzoneUtils.java | 227 +++
.../hadoop/ozone/web/utils/package-info.java | 18 +
.../main/proto/KeySpaceManagerProtocol.proto | 457 ++++++
.../src/main/shellprofile.d/hadoop-ozone.sh | 46 +
.../apache/hadoop/ozone/web/TestBucketInfo.java | 81 ++
.../hadoop/ozone/web/TestOzoneHttpServer.java | 98 ++
.../org/apache/hadoop/ozone/web/TestQuota.java | 116 ++
.../org/apache/hadoop/ozone/web/TestUtils.java | 101 ++
.../hadoop/ozone/web/TestVolumeStructs.java | 73 +
.../apache/hadoop/ozone/web/package-info.java | 21 +
hadoop-ozone/integration-test/pom.xml | 117 ++
.../hadoop/ozone/MiniOzoneClassicCluster.java | 615 ++++++++
.../apache/hadoop/ozone/MiniOzoneCluster.java | 62 +
.../hadoop/ozone/MiniOzoneTestHelper.java | 60 +
.../apache/hadoop/ozone/RatisTestHelper.java | 115 ++
.../hadoop/ozone/TestContainerOperations.java | 90 ++
.../hadoop/ozone/TestMiniOzoneCluster.java | 254 ++++
.../ozone/TestOzoneConfigurationFields.java | 38 +
.../apache/hadoop/ozone/TestOzoneHelper.java | 414 ++++++
.../ozone/TestStorageContainerManager.java | 447 ++++++
.../TestStorageContainerManagerHelper.java | 180 +++
.../ozone/client/rest/TestOzoneRestClient.java | 405 ++++++
.../hadoop/ozone/client/rest/package-info.java | 23 +
.../ozone/client/rpc/TestOzoneRpcClient.java | 815 +++++++++++
.../hadoop/ozone/client/rpc/package-info.java | 23 +
.../ozone/container/ContainerTestHelper.java | 595 ++++++++
.../common/TestBlockDeletingService.java | 398 ++++++
.../TestContainerDeletionChoosingPolicy.java | 197 +++
.../common/impl/TestContainerPersistence.java | 905 ++++++++++++
.../TestCloseContainerHandler.java | 113 ++
.../container/metrics/TestContainerMetrics.java | 124 ++
.../container/ozoneimpl/TestOzoneContainer.java | 584 ++++++++
.../ozoneimpl/TestOzoneContainerRatis.java | 143 ++
.../container/ozoneimpl/TestRatisManager.java | 129 ++
.../container/server/TestContainerServer.java | 258 ++++
.../ozone/ksm/TestContainerReportWithKeys.java | 142 ++
.../apache/hadoop/ozone/ksm/TestKSMMetrcis.java | 307 ++++
.../apache/hadoop/ozone/ksm/TestKSMSQLCli.java | 273 ++++
.../hadoop/ozone/ksm/TestKeySpaceManager.java | 1238 +++++++++++++++++
.../ksm/TestKeySpaceManagerRestInterface.java | 140 ++
.../ozone/ksm/TestKsmBlockVersioning.java | 254 ++++
.../ksm/TestMultipleContainerReadWrite.java | 216 +++
.../hadoop/ozone/ozShell/TestOzoneShell.java | 804 +++++++++++
.../hadoop/ozone/scm/TestAllocateContainer.java | 104 ++
.../hadoop/ozone/scm/TestContainerSQLCli.java | 300 ++++
.../ozone/scm/TestContainerSmallFile.java | 151 ++
.../org/apache/hadoop/ozone/scm/TestSCMCli.java | 535 +++++++
.../apache/hadoop/ozone/scm/TestSCMMXBean.java | 162 +++
.../apache/hadoop/ozone/scm/TestSCMMetrics.java | 251 ++++
.../ozone/scm/TestXceiverClientManager.java | 196 +++
.../ozone/scm/TestXceiverClientMetrics.java | 172 +++
.../BenchmarkContainerStateMap.java | 180 +++
.../ContainerStates/TestContainerStateMap.java | 226 +++
.../container/TestContainerStateManager.java | 303 ++++
.../hadoop/ozone/scm/node/TestQueryNode.java | 122 ++
.../hadoop/ozone/tools/TestDataValidate.java | 146 ++
.../apache/hadoop/ozone/tools/TestFreon.java | 136 ++
.../apache/hadoop/ozone/tools/package-info.java | 21 +
.../ozone/web/TestDistributedOzoneVolumes.java | 186 +++
.../hadoop/ozone/web/TestLocalOzoneVolumes.java | 185 +++
.../ozone/web/TestOzoneRestWithMiniCluster.java | 258 ++++
.../hadoop/ozone/web/TestOzoneWebAccess.java | 126 ++
.../hadoop/ozone/web/client/TestBuckets.java | 251 ++++
.../ozone/web/client/TestBucketsRatis.java | 76 +
.../hadoop/ozone/web/client/TestKeys.java | 692 +++++++++
.../hadoop/ozone/web/client/TestKeysRatis.java | 126 ++
.../ozone/web/client/TestOzoneClient.java | 306 ++++
.../hadoop/ozone/web/client/TestVolume.java | 434 ++++++
.../ozone/web/client/TestVolumeRatis.java | 131 ++
.../src/test/resources/webapps/ksm/.gitkeep | 15 +
.../src/test/resources/webapps/scm/.gitkeep | 15 +
hadoop-ozone/objectstore-service/pom.xml | 67 +
.../server/datanode/ObjectStoreHandler.java | 191 +++
.../hdfs/server/datanode/package-info.java | 22 +
.../org/apache/hadoop/ozone/OzoneRestUtils.java | 222 +++
.../org/apache/hadoop/ozone/package-info.java | 22 +
.../ozone/web/ObjectStoreApplication.java | 59 +
.../hadoop/ozone/web/ObjectStoreRestPlugin.java | 113 ++
.../hadoop/ozone/web/exceptions/ErrorTable.java | 225 +++
.../ozone/web/exceptions/package-info.java | 22 +
.../ozone/web/handlers/BucketHandler.java | 197 +++
.../web/handlers/BucketProcessTemplate.java | 323 +++++
.../hadoop/ozone/web/handlers/KeyHandler.java | 245 ++++
.../ozone/web/handlers/KeyProcessTemplate.java | 235 ++++
.../ozone/web/handlers/ServiceFilter.java | 61 +
.../web/handlers/StorageHandlerBuilder.java | 76 +
.../ozone/web/handlers/UserHandlerBuilder.java | 75 +
.../ozone/web/handlers/VolumeHandler.java | 272 ++++
.../web/handlers/VolumeProcessTemplate.java | 276 ++++
.../hadoop/ozone/web/handlers/package-info.java | 22 +
.../hadoop/ozone/web/interfaces/Accounting.java | 57 +
.../hadoop/ozone/web/interfaces/Bucket.java | 183 +++
.../hadoop/ozone/web/interfaces/Keys.java | 146 ++
.../ozone/web/interfaces/StorageHandler.java | 295 ++++
.../hadoop/ozone/web/interfaces/UserAuth.java | 101 ++
.../hadoop/ozone/web/interfaces/Volume.java | 182 +++
.../ozone/web/interfaces/package-info.java | 22 +
.../web/localstorage/LocalStorageHandler.java | 379 +++++
.../web/localstorage/OzoneMetadataManager.java | 1138 +++++++++++++++
.../ozone/web/localstorage/package-info.java | 18 +
.../LengthInputStreamMessageBodyWriter.java | 59 +
.../web/messages/StringMessageBodyWriter.java | 62 +
.../hadoop/ozone/web/messages/package-info.java | 18 +
.../web/netty/CloseableCleanupListener.java | 46 +
.../web/netty/ObjectStoreChannelHandler.java | 78 ++
.../web/netty/ObjectStoreJerseyContainer.java | 348 +++++
.../ObjectStoreJerseyContainerProvider.java | 40 +
.../web/netty/ObjectStoreRestHttpServer.java | 256 ++++
.../web/netty/ObjectStoreURLDispatcher.java | 61 +
...RequestContentObjectStoreChannelHandler.java | 117 ++
...equestDispatchObjectStoreChannelHandler.java | 103 ++
.../hadoop/ozone/web/netty/package-info.java | 26 +
.../apache/hadoop/ozone/web/package-info.java | 22 +
.../web/storage/DistributedStorageHandler.java | 539 ++++++++
.../hadoop/ozone/web/storage/package-info.java | 27 +
.../hadoop/ozone/web/userauth/Simple.java | 169 +++
.../hadoop/ozone/web/userauth/package-info.java | 23 +
...m.sun.jersey.spi.container.ContainerProvider | 16 +
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 +
hadoop-ozone/ozone-manager/pom.xml | 108 ++
.../apache/hadoop/ozone/ksm/BucketManager.java | 79 ++
.../hadoop/ozone/ksm/BucketManagerImpl.java | 315 +++++
.../org/apache/hadoop/ozone/ksm/KSMMXBean.java | 31 +
.../hadoop/ozone/ksm/KSMMetadataManager.java | 253 ++++
.../ozone/ksm/KSMMetadataManagerImpl.java | 525 +++++++
.../org/apache/hadoop/ozone/ksm/KSMMetrics.java | 437 ++++++
.../org/apache/hadoop/ozone/ksm/KSMStorage.java | 90 ++
.../hadoop/ozone/ksm/KeyDeletingService.java | 141 ++
.../org/apache/hadoop/ozone/ksm/KeyManager.java | 165 +++
.../apache/hadoop/ozone/ksm/KeyManagerImpl.java | 512 +++++++
.../hadoop/ozone/ksm/KeySpaceManager.java | 908 ++++++++++++
.../ozone/ksm/KeySpaceManagerHttpServer.java | 78 ++
.../hadoop/ozone/ksm/OpenKeyCleanupService.java | 116 ++
.../ozone/ksm/ServiceListJSONServlet.java | 103 ++
.../apache/hadoop/ozone/ksm/VolumeManager.java | 100 ++
.../hadoop/ozone/ksm/VolumeManagerImpl.java | 391 ++++++
.../ozone/ksm/exceptions/KSMException.java | 116 ++
.../ozone/ksm/exceptions/package-info.java | 19 +
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
...ceManagerProtocolServerSideTranslatorPB.java | 539 ++++++++
.../hadoop/ozone/protocolPB/package-info.java | 22 +
.../hadoop/ozone/web/ozShell/Handler.java | 79 ++
.../apache/hadoop/ozone/web/ozShell/Shell.java | 415 ++++++
.../web/ozShell/bucket/CreateBucketHandler.java | 96 ++
.../web/ozShell/bucket/DeleteBucketHandler.java | 87 ++
.../web/ozShell/bucket/InfoBucketHandler.java | 94 ++
.../web/ozShell/bucket/ListBucketHandler.java | 113 ++
.../web/ozShell/bucket/UpdateBucketHandler.java | 94 ++
.../ozone/web/ozShell/bucket/package-info.java | 23 +
.../web/ozShell/keys/DeleteKeyHandler.java | 96 ++
.../ozone/web/ozShell/keys/GetKeyHandler.java | 116 ++
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 98 ++
.../ozone/web/ozShell/keys/ListKeyHandler.java | 116 ++
.../ozone/web/ozShell/keys/PutKeyHandler.java | 106 ++
.../ozone/web/ozShell/keys/package-info.java | 23 +
.../hadoop/ozone/web/ozShell/package-info.java | 27 +
.../web/ozShell/volume/CreateVolumeHandler.java | 99 ++
.../web/ozShell/volume/DeleteVolumeHandler.java | 81 ++
.../web/ozShell/volume/InfoVolumeHandler.java | 95 ++
.../web/ozShell/volume/ListVolumeHandler.java | 114 ++
.../web/ozShell/volume/UpdateVolumeHandler.java | 92 ++
.../ozone/web/ozShell/volume/package-info.java | 23 +
.../apache/hadoop/ozone/web/package-info.java | 24 +
.../src/main/site/markdown/OzoneCommandShell.md | 150 ++
.../site/markdown/OzoneGettingStarted.md.vm | 320 +++++
.../src/main/site/markdown/OzoneMetrics.md | 166 +++
.../src/main/site/markdown/OzoneOverview.md | 88 ++
.../src/main/site/markdown/OzoneRest.md | 549 ++++++++
.../src/main/webapps/ksm/index.html | 70 +
.../src/main/webapps/ksm/ksm-metrics.html | 44 +
.../ozone-manager/src/main/webapps/ksm/ksm.js | 108 ++
.../ozone-manager/src/main/webapps/ksm/main.css | 23 +
.../src/main/webapps/ksm/main.html | 18 +
.../hadoop/ozone/ksm/TestBucketManagerImpl.java | 395 ++++++
.../hadoop/ozone/ksm/TestChunkStreams.java | 234 ++++
.../ksm/TestKeySpaceManagerHttpServer.java | 141 ++
.../apache/hadoop/ozone/ksm/package-info.java | 21 +
hadoop-ozone/pom.xml | 100 ++
hadoop-ozone/tools/pom.xml | 47 +
.../org/apache/hadoop/ozone/scm/cli/SQLCLI.java | 720 ++++++++++
.../hadoop/ozone/scm/cli/package-info.java | 22 +
.../org/apache/hadoop/ozone/tools/Freon.java | 1146 +++++++++++++++
.../apache/hadoop/ozone/tools/package-info.java | 22 +
.../org/apache/hadoop/test/OzoneTestDriver.java | 59 +
hadoop-project/pom.xml | 116 ++
hadoop-tools/hadoop-ozone/pom.xml | 9 +-
.../hadoop/fs/ozone/TestOzoneFSInputStream.java | 2 +-
.../fs/ozone/TestOzoneFileInterfaces.java | 235 ----
.../contract/ITestOzoneContractCreate.java | 48 -
.../contract/ITestOzoneContractDelete.java | 48 -
.../contract/ITestOzoneContractDistCp.java | 50 -
.../ITestOzoneContractGetFileStatus.java | 61 -
.../ozone/contract/ITestOzoneContractMkdir.java | 48 -
.../ozone/contract/ITestOzoneContractOpen.java | 47 -
.../contract/ITestOzoneContractRename.java | 49 -
.../contract/ITestOzoneContractRootDir.java | 51 -
.../ozone/contract/ITestOzoneContractSeek.java | 47 -
.../hadoop/fs/ozone/contract/OzoneContract.java | 125 --
.../src/test/resources/contract/ozone.xml | 113 --
.../src/test/resources/log4j.properties | 23 -
.../fs/ozone/TestOzoneFileInterfaces.java | 235 ++++
.../contract/ITestOzoneContractCreate.java | 48 +
.../contract/ITestOzoneContractDelete.java | 48 +
.../contract/ITestOzoneContractDistCp.java | 50 +
.../ITestOzoneContractGetFileStatus.java | 61 +
.../ozone/contract/ITestOzoneContractMkdir.java | 48 +
.../ozone/contract/ITestOzoneContractOpen.java | 47 +
.../contract/ITestOzoneContractRename.java | 49 +
.../contract/ITestOzoneContractRootDir.java | 51 +
.../ozone/contract/ITestOzoneContractSeek.java | 47 +
.../hadoop/fs/ozone/contract/OzoneContract.java | 125 ++
.../src/todo/resources/contract/ozone.xml | 113 ++
.../src/todo/resources/log4j.properties | 23 +
hadoop-tools/hadoop-tools-dist/pom.xml | 2 +-
.../yarn/api/records/ContainerRetryContext.java | 21 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../distributedshell/ApplicationMaster.java | 10 +-
.../applications/distributedshell/Client.java | 7 +
.../impl/pb/ContainerRetryContextPBImpl.java | 15 +
.../container/ContainerImpl.java | 86 +-
.../container/SlidingWindowRetryPolicy.java | 165 +++
.../recovery/NMLeveldbStateStoreService.java | 24 +
.../recovery/NMNullStateStoreService.java | 5 +
.../recovery/NMStateStoreService.java | 21 +
.../container/TestContainer.java | 37 +
.../container/TestSlidingWindowRetryPolicy.java | 77 ++
.../recovery/NMMemoryStateStoreService.java | 9 +
.../TestNMLeveldbStateStoreService.java | 16 +
.../UserGroupMappingPlacementRule.java | 7 +
.../server/resourcemanager/TestAppManager.java | 4 +-
.../TestUserGroupMappingPlacementRule.java | 21 +
.../capacity/TestCapacityScheduler.java | 34 +
...stCapacitySchedulerAutoCreatedQueueBase.java | 47 +-
.../TestCapacitySchedulerAutoQueueCreation.java | 43 +-
.../TestQueueManagementDynamicEditPolicy.java | 4 +-
pom.xml | 3 +
1372 files changed, 110389 insertions(+), 105816 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/dev-support/bin/dist-layout-stitching
----------------------------------------------------------------------
diff --git a/dev-support/bin/dist-layout-stitching b/dev-support/bin/dist-layout-stitching
index fdbc83c..5579435 100755
--- a/dev-support/bin/dist-layout-stitching
+++ b/dev-support/bin/dist-layout-stitching
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
# project.version
VERSION=$1
@@ -129,6 +128,24 @@ run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}"
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-nfs/target/hadoop-hdfs-nfs-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-client/target/hadoop-hdfs-client-${VERSION}" .
run copy "${ROOT}/hadoop-hdfs-project/hadoop-hdfs-native-client/target/hadoop-hdfs-native-client-${VERSION}" .
+
+run copy "${ROOT}/hadoop-hdsl/common/target/hadoop-hdsl-common-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/framework/target/hadoop-hdsl-server-framework-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/server-scm/target/hadoop-hdsl-server-scm-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/tools/target/hadoop-hdsl-tools-${VERSION}" .
+run copy "${ROOT}/hadoop-hdsl/container-service/target/hadoop-hdsl-container-service-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/common/target/hadoop-ozone-common-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/ozone-manager/target/hadoop-ozone-ozone-manager-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/ozone-client/target/hadoop-ozone-client-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/tools/target/hadoop-ozone-tools-${VERSION}" .
+run copy "${ROOT}/hadoop-ozone/objectstore-service/target/hadoop-ozone-objectstore-service-${VERSION}" .
+run copy "${ROOT}/hadoop-cblock/server/target/hadoop-cblock-server-${VERSION}" .
+run copy "${ROOT}/hadoop-cblock/tools/target/hadoop-cblock-tools-${VERSION}" .
+run cp -r "${ROOT}/hadoop-hdsl/framework/target/hadoop-hdsl-server-framework-${VERSION}/share/hadoop/hdsl/webapps/static" share/hadoop/ozone/webapps/
+run cp -r "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}/share/hadoop/hdfs/webapps/static" share/hadoop/ozone/webapps/
+run cp -r "${ROOT}/hadoop-hdfs-project/hadoop-hdfs/target/hadoop-hdfs-${VERSION}/share/hadoop/hdfs/webapps/static" share/hadoop/hdsl/webapps/
+
+
run copy "${ROOT}/hadoop-yarn-project/target/hadoop-yarn-project-${VERSION}" .
run copy "${ROOT}/hadoop-mapreduce-project/target/hadoop-mapreduce-${VERSION}" .
run copy "${ROOT}/hadoop-tools/hadoop-tools-dist/target/hadoop-tools-dist-${VERSION}" .
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/pom.xml b/hadoop-cblock/pom.xml
new file mode 100644
index 0000000..c6eb33e
--- /dev/null
+++ b/hadoop-cblock/pom.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project-dist</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <relativePath>../hadoop-project-dist</relativePath>
+ </parent>
+ <artifactId>hadoop-cblock</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <description>Apache Hadoop Cblock parent project</description>
+ <name>Apache Hadoop Cblock</name>
+ <packaging>pom</packaging>
+
+ <modules>
+ <module>server</module>
+ <module>tools</module>
+ </modules>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <excludeFilterFile combine.self="override"></excludeFilterFile>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml b/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
new file mode 100644
index 0000000..54d602c
--- /dev/null
+++ b/hadoop-cblock/server/dev-support/findbugsExcludeFile.xml
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+ <Match>
+ <Package name="org.apache.hadoop.cblock.protocol.proto"/>
+ </Match>
+</FindBugsFilter>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
deleted file mode 100644
index 0cc1482..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/Ozone.proto
+++ /dev/null
@@ -1,161 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "OzoneProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs.ozone;
-import "hdfs.proto";
-
-message PipelineChannel {
- required string leaderID = 1;
- repeated DatanodeIDProto members = 2;
- optional LifeCycleState state = 3 [default = OPEN];
- optional ReplicationType type = 4 [default = STAND_ALONE];
- optional ReplicationFactor factor = 5 [default = ONE];
- optional string name = 6;
-}
-
-// A pipeline is composed of PipelineChannel (Ratis/StandAlone) that back a
-// container.
-message Pipeline {
- required string containerName = 1;
- required PipelineChannel pipelineChannel = 2;
-}
-
-message KeyValue {
- required string key = 1;
- optional string value = 2;
-}
-
-/**
- * Type of the node.
- */
-enum NodeType {
- KSM = 1;
- SCM = 2;
- DATANODE = 3;
-}
-
-// Should we rename NodeState to DatanodeState?
-/**
- * Enum that represents the Node State. This is used in calls to getNodeList
- * and getNodeCount.
- */
-enum NodeState {
- HEALTHY = 1;
- STALE = 2;
- DEAD = 3;
- DECOMMISSIONING = 4;
- DECOMMISSIONED = 5;
- RAFT_MEMBER = 6;
- FREE_NODE = 7; // Not a member in raft.
- UNKNOWN = 8;
-}
-
-enum QueryScope {
- CLUSTER = 1;
- POOL = 2;
-}
-
-message Node {
- required DatanodeIDProto nodeID = 1;
- repeated NodeState nodeStates = 2;
-}
-
-message NodePool {
- repeated Node nodes = 1;
-}
-
-/**
- * LifeCycleState for SCM object creation state machine:
- * ->Allocated: allocated on SCM but clean has not started creating it yet.
- * ->Creating: allocated and assigned to client to create but not ack-ed yet.
- * ->Open: allocated on SCM and created on datanodes and ack-ed by a client.
- * ->Close: container closed due to space all used or error?
- * ->Timeout -> container failed to create on datanodes or ack-ed by client.
- * ->Deleting(TBD) -> container will be deleted after timeout
- * 1. ALLOCATE-ed containers on SCM can't serve key/block related operation
- * until ACK-ed explicitly which changes the state to OPEN.
- * 2. Only OPEN/CLOSED containers can serve key/block related operation.
- * 3. ALLOCATE-ed containers that are not ACK-ed timely will be TIMEOUT and
- * CLEANUP asynchronously.
- */
-
-enum LifeCycleState {
- ALLOCATED = 1;
- CREATING = 2; // Used for container allocated/created by different client.
- OPEN =3; // Mostly an update to SCM via HB or client call.
- CLOSING = 4;
- CLOSED = 5; // !!State after this has not been used yet.
- DELETING = 6;
- DELETED = 7; // object is deleted.
-}
-
-enum LifeCycleEvent {
- CREATE = 1; // A request to client to create this object
- CREATED = 2;
- FINALIZE = 3;
- CLOSE = 4; // !!Event after this has not been used yet.
- UPDATE = 5;
- TIMEOUT = 6; // creation has timed out from SCM's View.
- DELETE = 7;
- CLEANUP = 8;
-}
-
-message SCMContainerInfo {
- // TODO : Remove the container name from pipeline.
- required string containerName = 1;
- required LifeCycleState state = 2;
- required Pipeline pipeline = 3;
- // This is not total size of container, but space allocated by SCM for
- // clients to write blocks
- required uint64 allocatedBytes = 4;
- required uint64 usedBytes = 5;
- required uint64 numberOfKeys = 6;
- optional int64 stateEnterTime = 7;
- required string owner = 8;
- required int64 containerID = 9;
-}
-
-message GetScmInfoRequestProto {
-}
-
-message GetScmInfoRespsonseProto {
- required string clusterId = 1;
- required string scmId = 2;
-}
-
-
-enum ReplicationType {
- RATIS = 1;
- STAND_ALONE = 2;
- CHAINED = 3;
-}
-
-enum ReplicationFactor {
- ONE = 1;
- THREE = 3;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
deleted file mode 100644
index 99536ac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ScmBlockLocationProtocol.proto
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "ScmBlockLocationProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs;
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-
-// SCM Block protocol
-/**
- * keys - batch of block keys to find
- */
-message GetScmBlockLocationsRequestProto {
- repeated string keys = 1;
-}
-
-/**
- * locatedBlocks - for each requested hash, nodes that currently host the
- * container for that object key hash
- */
-message GetScmBlockLocationsResponseProto {
- repeated ScmLocatedBlockProto locatedBlocks = 1;
-}
-
-/**
- * Holds the nodes that currently host the blocks for a key.
- */
-message ScmLocatedBlockProto {
- required string key = 1;
- required hadoop.hdfs.ozone.Pipeline pipeline = 2;
-}
-
-/**
-* Request send to SCM asking allocate block of specified size.
-*/
-message AllocateScmBlockRequestProto {
- required uint64 size = 1;
- required hadoop.hdfs.ozone.ReplicationType type = 2;
- required hadoop.hdfs.ozone.ReplicationFactor factor = 3;
- required string owner = 4;
-
-}
-
-/**
- * A delete key request sent by KSM to SCM, it contains
- * multiple number of keys (and their blocks).
- */
-message DeleteScmKeyBlocksRequestProto {
- repeated KeyBlocks keyBlocks = 1;
-}
-
-/**
- * A object key and all its associated blocks.
- * We need to encapsulate object key name plus the blocks in this potocol
- * because SCM needs to response KSM with the keys it has deleted.
- * If the response only contains blocks, it will be very expensive for
- * KSM to figure out what keys have been deleted.
- */
-message KeyBlocks {
- required string key = 1;
- repeated string blocks = 2;
-}
-
-/**
- * A delete key response from SCM to KSM, it contains multiple child-results.
- * Each child-result represents a key deletion result, only if all blocks of
- * a key are successfully deleted, this key result is considered as succeed.
- */
-message DeleteScmKeyBlocksResponseProto {
- repeated DeleteKeyBlocksResultProto results = 1;
-}
-
-/**
- * A key deletion result. It contains all the block deletion results.
- */
-message DeleteKeyBlocksResultProto {
- required string objectKey = 1;
- repeated DeleteScmBlockResult blockResults = 2;
-}
-
-message DeleteScmBlockResult {
- enum Result {
- success = 1;
- chillMode = 2;
- errorNotFound = 3;
- unknownFailure = 4;
- }
- required Result result = 1;
- required string key = 2;
-}
-
-/**
- * Reply from SCM indicating that the container.
- */
-message AllocateScmBlockResponseProto {
- enum Error {
- success = 1;
- errorNotEnoughSpace = 2;
- errorSizeTooBig = 3;
- unknownFailure = 4;
- }
- required Error errorCode = 1;
- required string key = 2;
- required hadoop.hdfs.ozone.Pipeline pipeline = 3;
- required bool createContainer = 4;
- optional string errorMessage = 5;
-}
-
-/**
- * Protocol used from KeySpaceManager to StorageContainerManager.
- * See request and response messages for details of the RPC calls.
- */
-service ScmBlockLocationProtocolService {
-
- /**
- * Find the set of nodes that currently host the block, as
- * identified by the key. This method supports batch lookup by
- * passing multiple keys.
- */
- rpc getScmBlockLocations(GetScmBlockLocationsRequestProto)
- returns (GetScmBlockLocationsResponseProto);
-
- /**
- * Creates a block entry in SCM.
- */
- rpc allocateScmBlock(AllocateScmBlockRequestProto)
- returns (AllocateScmBlockResponseProto);
-
- /**
- * Deletes blocks for a set of object keys from SCM.
- */
- rpc deleteScmKeyBlocks(DeleteScmKeyBlocksRequestProto)
- returns (DeleteScmKeyBlocksResponseProto);
-
- /**
- * Gets the scmInfo from SCM.
- */
- rpc getScmInfo(hadoop.hdfs.ozone.GetScmInfoRequestProto)
- returns (hadoop.hdfs.ozone.GetScmInfoRespsonseProto);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
deleted file mode 100644
index 512ec75..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/StorageContainerLocationProtocol.proto
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * These .proto interfaces are private and unstable.
- * Please see http://wiki.apache.org/hadoop/Compatibility
- * for what changes are allowed for a *unstable* .proto interface.
- */
-
-option java_package = "org.apache.hadoop.ozone.protocol.proto";
-option java_outer_classname = "StorageContainerLocationProtocolProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.hdfs;
-
-import "hdfs.proto";
-import "Ozone.proto";
-
-/**
-* Request send to SCM asking where the container should be created.
-*/
-message ContainerRequestProto {
- required string containerName = 1;
- // Ozone only support replciation of either 1 or 3.
- required hadoop.hdfs.ozone.ReplicationFactor replicationFactor = 2;
- required hadoop.hdfs.ozone.ReplicationType replicationType = 3;
- required string owner = 4;
-
-}
-
-/**
- * Reply from SCM indicating that the container.
- */
-message ContainerResponseProto {
- enum Error {
- success = 1;
- errorContainerAlreadyExists = 2;
- errorContainerMissing = 3;
- }
- required Error errorCode = 1;
- required hadoop.hdfs.ozone.Pipeline pipeline = 2;
- optional string errorMessage = 3;
-}
-
-message GetContainerRequestProto {
- required string containerName = 1;
-}
-
-message GetContainerResponseProto {
- required hadoop.hdfs.ozone.Pipeline pipeline = 1;
-}
-
-message ListContainerRequestProto {
- required uint32 count = 1;
- optional string startName = 2;
- optional string prefixName = 3;
-}
-
-message ListContainerResponseProto {
- repeated hadoop.hdfs.ozone.SCMContainerInfo containers = 1;
-}
-
-message DeleteContainerRequestProto {
- required string containerName = 1;
-}
-
-message DeleteContainerResponseProto {
- // Empty response
-}
-
-message ObjectStageChangeRequestProto {
- enum Type {
- container = 1;
- pipeline = 2;
- }
- // delete/copy operation may be added later
- enum Op {
- create = 1;
- close = 2;
- }
- enum Stage {
- begin = 1;
- complete = 2;
- }
- required string name = 1;
- required Type type = 2;
- required Op op= 3;
- required Stage stage = 4;
-}
-
-message ObjectStageChangeResponseProto {
- // Empty response
-}
-
-/*
- NodeQueryRequest sends a request to SCM asking to send a list of nodes that
- match the NodeState that we are requesting.
-*/
-message NodeQueryRequestProto {
-
-
- // Repeated, So we can specify more than one status type.
- // These NodeState types are additive for now, in the sense that
- // if you specify HEALTHY and FREE_NODE members --
- // Then you get all healthy node which are not raft members.
- //
- // if you specify all healthy and dead nodes, you will get nothing
- // back. Server is not going to dictate what combinations make sense,
- // it is entirely up to the caller.
- // TODO: Support operators like OR and NOT. Currently it is always an
- // implied AND.
-
- repeated hadoop.hdfs.ozone.NodeState query = 1;
- required hadoop.hdfs.ozone.QueryScope scope = 2;
- optional string poolName = 3; // if scope is pool, then pool name is needed.
-}
-
-message NodeQueryResponseProto {
- required hadoop.hdfs.ozone.NodePool datanodes = 1;
-}
-
-/**
- Request to create a replication pipeline.
- */
-message PipelineRequestProto {
- required hadoop.hdfs.ozone.ReplicationType replicationType = 1;
- required hadoop.hdfs.ozone.ReplicationFactor replicationFactor = 2;
-
- // if datanodes are specified then pipelines are created using those
- // datanodes.
- optional hadoop.hdfs.ozone.NodePool nodePool = 3;
- optional string pipelineID = 4;
-}
-
-message PipelineResponseProto {
- enum Error {
- success = 1;
- errorPipelineAlreadyExists = 2;
- }
- required Error errorCode = 1;
- optional hadoop.hdfs.ozone.Pipeline pipeline = 2;
- optional string errorMessage = 3;
-}
-
-/**
- * Protocol used from an HDFS node to StorageContainerManager. See the request
- * and response messages for details of the RPC calls.
- */
-service StorageContainerLocationProtocolService {
-
- /**
- * Creates a container entry in SCM.
- */
- rpc allocateContainer(ContainerRequestProto) returns (ContainerResponseProto);
-
- /**
- * Returns the pipeline for a given container.
- */
- rpc getContainer(GetContainerRequestProto) returns (GetContainerResponseProto);
-
- rpc listContainer(ListContainerRequestProto) returns (ListContainerResponseProto);
-
- /**
- * Deletes a container in SCM.
- */
- rpc deleteContainer(DeleteContainerRequestProto) returns (DeleteContainerResponseProto);
-
- /**
- * Returns a set of Nodes that meet a criteria.
- */
- rpc queryNode(NodeQueryRequestProto) returns (NodeQueryResponseProto);
-
- /**
- * Notify from client when begin or finish container or pipeline operations on datanodes.
- */
- rpc notifyObjectStageChange(ObjectStageChangeRequestProto) returns (ObjectStageChangeResponseProto);
-
- /*
- * Apis that Manage Pipelines.
- *
- * Pipelines are abstractions offered by SCM and Datanode that allows users
- * to create a replication pipeline.
- *
- * These following APIs allow command line programs like SCM CLI to list
- * and manage pipelines.
- */
-
- /**
- * Creates a replication pipeline.
- */
- rpc allocatePipeline(PipelineRequestProto)
- returns (PipelineResponseProto);
-
- /**
- * Returns information about SCM.
- */
- rpc getScmInfo(hadoop.hdfs.ozone.GetScmInfoRequestProto)
- returns (hadoop.hdfs.ozone.GetScmInfoRespsonseProto);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index e9b5c24..4db2d58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -75,6 +75,7 @@ message DatanodeIDProto {
optional uint32 infoSecurePort = 7 [default = 0]; // datanode https port
optional uint32 containerPort = 8 [default = 0]; // Ozone stand_alone protocol
optional uint32 ratisPort = 9 [default = 0]; //Ozone ratis port
+ optional uint32 ozoneRestPort = 10 [default = 0];
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
deleted file mode 100644
index 03c45c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/TestOzoneAcls.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone;
-
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Set;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * This class is to test acl stoarge and retreival in ozone store.
- */
-public class TestOzoneAcls {
-
- @Test
- public void testAclParse() {
- HashMap<String, Boolean> testMatrix;
- testMatrix = new HashMap<>();
-
- testMatrix.put("user:bilbo:r", Boolean.TRUE);
- testMatrix.put("user:bilbo:w", Boolean.TRUE);
- testMatrix.put("user:bilbo:rw", Boolean.TRUE);
- testMatrix.put("user:bilbo:wr", Boolean.TRUE);
- testMatrix.put(" user:bilbo:wr ", Boolean.TRUE);
-
-
- // ACLs makes no judgement on the quality of
- // user names. it is for the userAuth interface
- // to determine if a user name is really a name
- testMatrix.put(" user:*:rw", Boolean.TRUE);
- testMatrix.put(" user:~!:rw", Boolean.TRUE);
-
-
- testMatrix.put("", Boolean.FALSE);
- testMatrix.put(null, Boolean.FALSE);
- testMatrix.put(" user:bilbo:", Boolean.FALSE);
- testMatrix.put(" user:bilbo:rx", Boolean.FALSE);
- testMatrix.put(" user:bilbo:mk", Boolean.FALSE);
- testMatrix.put(" user::rw", Boolean.FALSE);
- testMatrix.put("user11:bilbo:rw", Boolean.FALSE);
- testMatrix.put(" user:::rw", Boolean.FALSE);
-
- testMatrix.put(" group:hobbit:r", Boolean.TRUE);
- testMatrix.put(" group:hobbit:w", Boolean.TRUE);
- testMatrix.put(" group:hobbit:rw", Boolean.TRUE);
- testMatrix.put(" group:hobbit:wr", Boolean.TRUE);
- testMatrix.put(" group:*:rw", Boolean.TRUE);
- testMatrix.put(" group:~!:rw", Boolean.TRUE);
-
- testMatrix.put(" group:hobbit:", Boolean.FALSE);
- testMatrix.put(" group:hobbit:rx", Boolean.FALSE);
- testMatrix.put(" group:hobbit:mk", Boolean.FALSE);
- testMatrix.put(" group::", Boolean.FALSE);
- testMatrix.put(" group::rw", Boolean.FALSE);
- testMatrix.put(" group22:hobbit:", Boolean.FALSE);
- testMatrix.put(" group:::rw", Boolean.FALSE);
-
- testMatrix.put("JUNK group:hobbit:r", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:w", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:rw", Boolean.FALSE);
- testMatrix.put("JUNK group:hobbit:wr", Boolean.FALSE);
- testMatrix.put("JUNK group:*:rw", Boolean.FALSE);
- testMatrix.put("JUNK group:~!:rw", Boolean.FALSE);
-
- testMatrix.put(" world::r", Boolean.TRUE);
- testMatrix.put(" world::w", Boolean.TRUE);
- testMatrix.put(" world::rw", Boolean.TRUE);
- testMatrix.put(" world::wr", Boolean.TRUE);
-
- testMatrix.put(" world:bilbo:w", Boolean.FALSE);
- testMatrix.put(" world:bilbo:rw", Boolean.FALSE);
-
- Set<String> keys = testMatrix.keySet();
- for (String key : keys) {
- if (testMatrix.get(key)) {
- OzoneAcl.parseAcl(key);
- } else {
- try {
- OzoneAcl.parseAcl(key);
- // should never get here since parseAcl will throw
- fail("An exception was expected but did not happen.");
- } catch (IllegalArgumentException e) {
- // nothing to do
- }
- }
- }
- }
-
- @Test
- public void testAclValues() {
- OzoneAcl acl = OzoneAcl.parseAcl("user:bilbo:rw");
- assertEquals(acl.getName(), "bilbo");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:wr");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:r");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.READ, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("user:bilbo:w");
- assertEquals("bilbo", acl.getName());
- assertEquals(OzoneAcl.OzoneACLRights.WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.USER, acl.getType());
-
- acl = OzoneAcl.parseAcl("group:hobbit:wr");
- assertEquals(acl.getName(), "hobbit");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.GROUP, acl.getType());
-
- acl = OzoneAcl.parseAcl("world::wr");
- assertEquals(acl.getName(), "");
- assertEquals(OzoneAcl.OzoneACLRights.READ_WRITE, acl.getRights());
- assertEquals(OzoneAcl.OzoneACLType.WORLD, acl.getType());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
deleted file mode 100644
index d4c626e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/ozone/common/TestStateMachine.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.common;
-
-import org.apache.commons.collections.SetUtils;
-import org.apache.hadoop.ozone.common.statemachine.InvalidStateTransitionException;
-import org.apache.hadoop.ozone.common.statemachine.StateMachine;
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.HashSet;
-import java.util.Set;
-
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.INIT;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CREATING;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.OPERATIONAL;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLOSED;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.CLEANUP;
-import static org.apache.hadoop.ozone.common.TestStateMachine.STATES.FINAL;
-
-/**
- * This class is to test ozone common state machine.
- */
-public class TestStateMachine {
-
- /**
- * STATES used by the test state machine.
- */
- public enum STATES {INIT, CREATING, OPERATIONAL, CLOSED, CLEANUP, FINAL};
-
- /**
- * EVENTS used by the test state machine.
- */
- public enum EVENTS {ALLOCATE, CREATE, UPDATE, CLOSE, DELETE, TIMEOUT};
-
- @Rule
- public ExpectedException exception = ExpectedException.none();
-
- @Test
- public void testStateMachineStates() throws InvalidStateTransitionException {
- Set<STATES> finals = new HashSet<>();
- finals.add(FINAL);
-
- StateMachine<STATES, EVENTS> stateMachine =
- new StateMachine<>(INIT, finals);
-
- stateMachine.addTransition(INIT, CREATING, EVENTS.ALLOCATE);
- stateMachine.addTransition(CREATING, OPERATIONAL, EVENTS.CREATE);
- stateMachine.addTransition(OPERATIONAL, OPERATIONAL, EVENTS.UPDATE);
- stateMachine.addTransition(OPERATIONAL, CLEANUP, EVENTS.DELETE);
- stateMachine.addTransition(OPERATIONAL, CLOSED, EVENTS.CLOSE);
- stateMachine.addTransition(CREATING, CLEANUP, EVENTS.TIMEOUT);
-
- // Initial and Final states
- Assert.assertEquals("Initial State", INIT, stateMachine.getInitialState());
- Assert.assertTrue("Final States", SetUtils.isEqualSet(finals,
- stateMachine.getFinalStates()));
-
- // Valid state transitions
- Assert.assertEquals("STATE should be OPERATIONAL after being created",
- OPERATIONAL, stateMachine.getNextState(CREATING, EVENTS.CREATE));
- Assert.assertEquals("STATE should be OPERATIONAL after being updated",
- OPERATIONAL, stateMachine.getNextState(OPERATIONAL, EVENTS.UPDATE));
- Assert.assertEquals("STATE should be CLEANUP after being deleted",
- CLEANUP, stateMachine.getNextState(OPERATIONAL, EVENTS.DELETE));
- Assert.assertEquals("STATE should be CLEANUP after being timeout",
- CLEANUP, stateMachine.getNextState(CREATING, EVENTS.TIMEOUT));
- Assert.assertEquals("STATE should be CLOSED after being closed",
- CLOSED, stateMachine.getNextState(OPERATIONAL, EVENTS.CLOSE));
-
- // Negative cases: invalid transition
- expectException();
- stateMachine.getNextState(OPERATIONAL, EVENTS.CREATE);
-
- expectException();
- stateMachine.getNextState(CREATING, EVENTS.CLOSE);
- }
-
- /**
- * We expect an InvalidStateTransitionException.
- */
- private void expectException() {
- exception.expect(InvalidStateTransitionException.class);
- exception.expectMessage("Invalid event");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
deleted file mode 100644
index 86adddb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/TestArchive.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.RandomUtils;
-import org.apache.hadoop.fs.FileUtil;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.Iterator;
-import java.util.Random;
-import java.util.zip.Adler32;
-import java.util.zip.Checksum;
-
-/**
- * Test archive creation and unpacking.
- */
-public class TestArchive {
- private static final int DIR_COUNT = 10;
- private static final int SUB_DIR_COUNT = 3;
- private static final int FILE_COUNT = 10;
- private long checksumWrite = 0L;
- private long checksumRead = 0L;
- private long tmp = 0L;
- private Checksum crc = new Adler32();
-
- @Rule
- public TemporaryFolder folder = new TemporaryFolder();
-
- @Rule
- public TemporaryFolder outputFolder = new TemporaryFolder();
-
-
- @Before
- public void setUp() throws Exception {
- Random r = new Random();
- final int megaByte = 1024 * 1024;
-
- for (int x = 0; x < DIR_COUNT; x++) {
- File subdir = folder.newFolder(String.format("dir%d", x));
- for (int y = 0; y < SUB_DIR_COUNT; y++) {
- File targetDir = new File(subdir.getPath().concat(File.separator)
- .concat(String.format("subdir%d%d", x, y)));
- if(!targetDir.mkdirs()) {
- throw new IOException("Failed to create subdirectory. " +
- targetDir.toString());
- }
- for (int z = 0; z < FILE_COUNT; z++) {
- Path temp = Paths.get(targetDir.getPath().concat(File.separator)
- .concat(String.format("File%d.txt", z)));
- byte[] buf = RandomUtils.nextBytes(r.nextInt(megaByte));
- Files.write(temp, buf);
- crc.reset();
- crc.update(buf, 0, buf.length);
- tmp = crc.getValue();
- checksumWrite +=tmp;
- }
- }
- }
- }
-
- @Test
- public void testArchive() throws Exception {
- File archiveFile = new File(outputFolder.getRoot() + File.separator
- + "test.container.zip");
- long zipCheckSum = FileUtil.zip(folder.getRoot(), archiveFile);
- Assert.assertTrue(zipCheckSum > 0);
- File decomp = new File(outputFolder.getRoot() + File.separator +
- "decompress");
- if (!decomp.exists() && !decomp.mkdirs()) {
- throw new IOException("Unable to create the destination directory. " +
- decomp.getPath());
- }
-
- FileUtil.unZip(archiveFile, decomp);
- String[] patterns = {"txt"};
- Iterator<File> iter = FileUtils.iterateFiles(decomp, patterns, true);
- int count = 0;
- while (iter.hasNext()) {
- count++;
- byte[] buf = Files.readAllBytes(iter.next().toPath());
- crc.reset();
- crc.update(buf, 0, buf.length);
- tmp = crc.getValue();
- checksumRead += tmp;
- }
- Assert.assertEquals(DIR_COUNT * SUB_DIR_COUNT * FILE_COUNT, count);
- Assert.assertEquals(checksumWrite, checksumRead);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
deleted file mode 100644
index 9c480d6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/test/java/org/apache/hadoop/scm/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-/**
- Test cases for SCM client classes.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
index b4da6ba7..a99ea75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/dev-support/findbugsExcludeFile.xml
@@ -3,9 +3,6 @@
<Package name="org.apache.hadoop.record.compiler.generated" />
</Match>
<Match>
- <Package name="org.apache.hadoop.hdfs.ozone.protocol.proto" />
- </Match>
- <Match>
<Package name="org.apache.hadoop.hdfs.protocol.proto" />
</Match>
<Match>
@@ -18,12 +15,6 @@
<Package name="org.apache.hadoop.hdfs.qjournal.protocol" />
</Match>
<Match>
- <Package name="org.apache.hadoop.ozone.protocol.proto" />
- </Match>
- <Match>
- <Package name ="org.apache.hadoop.cblock.protocol.proto" />
- </Match>
- <Match>
<Package name="org.apache.hadoop.hdfs.federation.protocol.proto" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index c57ecc0..909e589 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -169,6 +169,11 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-all</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.htrace</groupId>
<artifactId>htrace-core4</artifactId>
</dependency>
@@ -187,17 +192,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>org.fusesource.leveldbjni</groupId>
<artifactId>leveldbjni-all</artifactId>
</dependency>
- <dependency>
- <groupId>org.rocksdb</groupId>
- <artifactId>rocksdbjni</artifactId>
- <version>5.8.0</version>
- </dependency>
- <dependency>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
- <version>1.5.9</version>
- <scope>provided</scope>
- </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.bouncycastle</groupId>
@@ -209,68 +203,15 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<artifactId>jackson-databind</artifactId>
</dependency>
<dependency>
- <groupId>org.jscsi</groupId>
- <artifactId>target</artifactId>
- <version>2.6.0</version>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>ch.qos.logback</groupId>
- <artifactId>logback-classic</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.jctools</groupId>
- <artifactId>jctools-core</artifactId>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>org.xerial</groupId>
- <artifactId>sqlite-jdbc</artifactId>
- <version>3.8.7</version>
- </dependency>
- <dependency>
- <groupId>io.kubernetes</groupId>
- <artifactId>client-java</artifactId>
- <version>1.0.0-beta1</version>
- <exclusions>
- <exclusion>
- <groupId>io.swagger</groupId>
- <artifactId>swagger-annotations</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.github.stefanbirkner</groupId>
- <artifactId>system-rules</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>io.dropwizard.metrics</groupId>
- <artifactId>metrics-core</artifactId>
- </dependency>
<dependency>
<groupId>org.assertj</groupId>
<artifactId>assertj-core</artifactId>
<scope>test</scope>
</dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-core</artifactId>
- <version>1.19</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.openjdk.jmh</groupId>
- <artifactId>jmh-generator-annprocess</artifactId>
- <version>1.19</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
@@ -366,7 +307,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<configuration>
<tasks>
<copy file="src/main/resources/hdfs-default.xml" todir="src/site/resources"/>
- <copy file="src/main/resources/ozone-default.xml" todir="src/site/resources"/>
<copy file="src/main/xsl/configuration.xsl" todir="src/site/resources"/>
</tasks>
</configuration>
@@ -403,9 +343,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<include>QJournalProtocol.proto</include>
<include>editlog.proto</include>
<include>fsimage.proto</include>
- <include>StorageContainerDatanodeProtocol.proto</include>
- <include>CBlockServiceProtocol.proto</include>
- <include>CBlockClientServerProtocol.proto</include>
<include>FederationProtocol.proto</include>
<include>RouterProtocol.proto</include>
<include>AliasMapProtocol.proto</include>
@@ -472,14 +409,6 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<exclude>src/main/webapps/static/dataTables.bootstrap.js</exclude>
<exclude>src/main/webapps/static/d3-v4.1.1.min.js</exclude>
<exclude>src/test/resources/diskBalancer/data-cluster-3node-3disk.json</exclude>
- <exclude>src/main/webapps/static/nvd3-1.8.5.min.css.map</exclude>
- <exclude>src/main/webapps/static/nvd3-1.8.5.min.js</exclude>
- <exclude>src/main/webapps/static/angular-route-1.6.4.min.js</exclude>
- <exclude>src/main/webapps/static/nvd3-1.8.5.min.css</exclude>
- <exclude>src/main/webapps/static/angular-nvd3-1.0.9.min.js</exclude>
- <exclude>src/main/webapps/static/nvd3-1.8.5.min.js.map</exclude>
- <exclude>src/main/webapps/static/angular-1.6.4.min.js</exclude>
- <exclude>src/main/webapps/static/d3-3.5.17.min.js</exclude>
</excludes>
</configuration>
</plugin>
@@ -492,55 +421,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<includes>
<include>configuration.xsl</include>
<include>hdfs-default.xml</include>
- <include>ozone-default.xml</include>
</includes>
<followSymlinks>false</followSymlinks>
</fileset>
</filesets>
</configuration>
</plugin>
- <plugin>
- <groupId>com.github.kongchen</groupId>
- <artifactId>swagger-maven-plugin</artifactId>
- <version>3.1.5</version>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>generate</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <apiSources>
- <apiSource>
- <springmvc>false</springmvc>
- <swaggerDirectory>target/webapps/static</swaggerDirectory>
- <swaggerFileName>ozone.swagger</swaggerFileName>
- <schemes>
- <scheme>http</scheme>
- </schemes>
- <host>localhost:9864</host>
- <basePath>/</basePath>
- <locations>
- <location>org.apache.hadoop.ozone.web.interfaces</location>
- </locations>
- <info>
- <title>HDFS Ozone REST Api</title>
- <version>${project.version}</version>
- <contact>
- <name>Apache Hadoop project</name>
- <url>https://hadoop.apache.org</url>
- </contact>
- <license>
- <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
- <name>Apache 2.0</name>
- </license>
- </info>
- </apiSource>
- </apiSources>
- </configuration>
- </plugin>
</plugins>
</build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 4be674b..38be348b 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -32,11 +32,8 @@ function hadoop_usage
hadoop_add_option "--hosts filename" "list of hosts to use in worker mode"
hadoop_add_option "--workers" "turn on worker mode"
-
hadoop_add_subcommand "balancer" daemon "run a cluster balancing utility"
hadoop_add_subcommand "cacheadmin" admin "configure the HDFS cache"
- hadoop_add_subcommand "cblock" admin "cblock CLI"
- hadoop_add_subcommand "cblockserver" daemon "run cblock server"
hadoop_add_subcommand "classpath" client "prints the class path needed to get the hadoop jar and the required libraries"
hadoop_add_subcommand "crypto" admin "configure HDFS encryption zones"
hadoop_add_subcommand "datanode" daemon "run a DFS datanode"
@@ -49,15 +46,12 @@ function hadoop_usage
hadoop_add_subcommand "envvars" client "display computed Hadoop environment variables"
hadoop_add_subcommand "ec" admin "run a HDFS ErasureCoding CLI"
hadoop_add_subcommand "fetchdt" client "fetch a delegation token from the NameNode"
- hadoop_add_subcommand "freon" client "runs an ozone data generator"
hadoop_add_subcommand "fsck" admin "run a DFS filesystem checking utility"
hadoop_add_subcommand "getconf" client "get config values from configuration"
hadoop_add_subcommand "groups" client "get the groups which users belong to"
hadoop_add_subcommand "haadmin" admin "run a DFS HA admin client"
hadoop_add_subcommand "jmxget" admin "get JMX exported values from NameNode or DataNode."
hadoop_add_subcommand "journalnode" daemon "run the DFS journalnode"
- hadoop_add_subcommand "jscsi" daemon "run cblock jscsi server"
- hadoop_add_subcommand "ksm" daemon "Ozone keyspace manager"
hadoop_add_subcommand "lsSnapshottableDir" client "list all snapshottable dirs owned by the current user"
hadoop_add_subcommand "mover" daemon "run a utility to move block replicas across storage types"
hadoop_add_subcommand "namenode" daemon "run the DFS namenode"
@@ -65,17 +59,12 @@ function hadoop_usage
hadoop_add_subcommand "oev" admin "apply the offline edits viewer to an edits file"
hadoop_add_subcommand "oiv" admin "apply the offline fsimage viewer to an fsimage"
hadoop_add_subcommand "oiv_legacy" admin "apply the offline fsimage viewer to a legacy fsimage"
- hadoop_add_subcommand "oz" client "command line interface for ozone"
- hadoop_add_subcommand "oz_debug" client "ozone debug tool, convert ozone metadata into relational data"
hadoop_add_subcommand "portmap" daemon "run a portmap service"
- hadoop_add_subcommand "scm" daemon "run the Storage Container Manager service"
- hadoop_add_subcommand "scmcli" client "run the CLI of the Storage Container Manager "
hadoop_add_subcommand "secondarynamenode" daemon "run the DFS secondary namenode"
hadoop_add_subcommand "snapshotDiff" client "diff two snapshots of a directory or diff the current directory contents with a snapshot"
hadoop_add_subcommand "storagepolicies" admin "list/get/set block storage policies"
hadoop_add_subcommand "version" client "print the version"
hadoop_add_subcommand "zkfc" daemon "run the ZK Failover Controller daemon"
-
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" false
}
@@ -97,13 +86,6 @@ function hdfscmd_case
cacheadmin)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
;;
- cblock)
- HADOOP_CLASSNAME=org.apache.hadoop.cblock.cli.CBlockCli
- ;;
- cblockserver)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME=org.apache.hadoop.cblock.CBlockManager
- ;;
classpath)
hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@"
;;
@@ -157,9 +139,6 @@ function hdfscmd_case
fetchdt)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
;;
- freon)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.tools.Freon
- ;;
fsck)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
;;
@@ -176,17 +155,9 @@ function hdfscmd_case
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
;;
- jscsi)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME=org.apache.hadoop.cblock.jscsiHelper.SCSITargetDaemon
- ;;
jmxget)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
;;
- ksm)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.ksm.KeySpaceManager
- ;;
lsSnapshottableDir)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir
;;
@@ -215,28 +186,10 @@ function hdfscmd_case
oiv_legacy)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
;;
- oz)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.web.ozShell.Shell
- ;;
- oz_debug)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SQLCLI
- ;;
portmap)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
;;
- scmcli)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SCMCLI
- ;;
- scm)
- HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
- HADOOP_CLASSNAME='org.apache.hadoop.ozone.scm.StorageContainerManager'
- hadoop_debug "Appending HDFS_STORAGECONTAINERMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HDFS_STORAGECONTAINERMANAGER_OPTS}"
- ;;
- scmcli)
- HADOOP_CLASSNAME=org.apache.hadoop.ozone.scm.cli.SCMCLI
- ;;
secondarynamenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
index bde4b3a..df044fe 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-balancer.sh
@@ -17,10 +17,6 @@
MYNAME="${BASH_SOURCE-$0}"
-## @description usage info
-## @audience private
-## @stability evolving
-## @replaceable no
function hadoop_usage
{
hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
deleted file mode 100644
index 097490e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/start-ozone.sh
+++ /dev/null
@@ -1,105 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Start hadoop hdfs and ozone daemons.
-# Run this on master node.
-## @description usage info
-## @audience private
-## @stability evolving
-## @replaceable no
-function hadoop_usage
-{
- echo "Usage: start-ozone.sh"
-}
-
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
- # shellcheck disable=SC1090
- . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
- exit 1
-fi
-
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
-
-#---------------------------------------------------------
-# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ "${OZONE_ENABLED}" != "true" ]]; then
- echo "Operation is not supported because ozone is not enabled."
- exit -1
-fi
-
-#---------------------------------------------------------
-# Start hdfs before starting ozone daemons
-if [[ -f "${bin}/start-dfs.sh" ]]; then
- "${bin}/start-dfs.sh"
-else
- echo "ERROR: Cannot execute ${bin}/start-dfs.sh." 2>&1
- exit 1
-fi
-
-#---------------------------------------------------------
-# Ozone keyspacemanager nodes
-KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -keyspacemanagers 2>/dev/null)
-echo "Starting key space manager nodes [${KSM_NODES}]"
-if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
- KSM_NODES=$(hostname)
-fi
-
-hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${KSM_NODES}" \
- --daemon start \
- ksm
-
-HADOOP_JUMBO_RETCOUNTER=$?
-
-#---------------------------------------------------------
-# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -storagecontainermanagers 2>/dev/null)
-echo "Starting storage container manager nodes [${SCM_NODES}]"
-hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${SCM_NODES}" \
- --daemon start \
- scm
-
-(( HADOOP_JUMBO_RETCOUNTER=HADOOP_JUMBO_RETCOUNTER + $? ))
-
-exit ${HADOOP_JUMBO_RETCOUNTER}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
index 487aee8..ec94080 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-balancer.sh
@@ -17,10 +17,6 @@
MYNAME="${BASH_SOURCE-$0}"
-## @description usage info
-## @audience private
-## @stability evolving
-## @replaceable no
function hadoop_usage
{
hadoop_add_option "--buildpaths" "attempt to add class files from build tree"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
deleted file mode 100644
index bf7309d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/stop-ozone.sh
+++ /dev/null
@@ -1,99 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Stop hdfs and ozone daemons.
-# Run this on master node.
-## @description usage info
-## @audience private
-## @stability evolving
-## @replaceable no
-function hadoop_usage
-{
- echo "Usage: stop-ozone.sh"
-}
-
-this="${BASH_SOURCE-$0}"
-bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P)
-
-# let's locate libexec...
-if [[ -n "${HADOOP_HOME}" ]]; then
- HADOOP_DEFAULT_LIBEXEC_DIR="${HADOOP_HOME}/libexec"
-else
- HADOOP_DEFAULT_LIBEXEC_DIR="${bin}/../libexec"
-fi
-
-HADOOP_LIBEXEC_DIR="${HADOOP_LIBEXEC_DIR:-$HADOOP_DEFAULT_LIBEXEC_DIR}"
-# shellcheck disable=SC2034
-HADOOP_NEW_CONFIG=true
-if [[ -f "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh" ]]; then
- # shellcheck disable=SC1090
- . "${HADOOP_LIBEXEC_DIR}/hdfs-config.sh"
-else
- echo "ERROR: Cannot execute ${HADOOP_LIBEXEC_DIR}/hdfs-config.sh." 2>&1
- exit 1
-fi
-
-SECURITY_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authentication | tr '[:upper:]' '[:lower:]' 2>&-)
-SECURITY_AUTHORIZATION_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey hadoop.security.authorization | tr '[:upper:]' '[:lower:]' 2>&-)
-
-if [[ ${SECURITY_ENABLED} == "kerberos" || ${SECURITY_AUTHORIZATION_ENABLED} == "true" ]]; then
- echo "Ozone is not supported in a security enabled cluster."
- exit 1
-fi
-
-#---------------------------------------------------------
-# Check if ozone is enabled
-OZONE_ENABLED=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -confKey ozone.enabled | tr '[:upper:]' '[:lower:]' 2>&-)
-if [[ "${OZONE_ENABLED}" != "true" ]]; then
- echo "Operation is not supported because ozone is not enabled."
- exit -1
-fi
-
-#---------------------------------------------------------
-# Start hdfs before starting ozone daemons
-if [[ -f "${bin}/stop-dfs.sh" ]]; then
- "${bin}/stop-dfs.sh"
-else
- echo "ERROR: Cannot execute ${bin}/stop-dfs.sh." 2>&1
- exit 1
-fi
-
-#---------------------------------------------------------
-# Ozone keyspacemanager nodes
-KSM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -keyspacemanagers 2>/dev/null)
-echo "Stopping key space manager nodes [${KSM_NODES}]"
-if [[ "${KSM_NODES}" == "0.0.0.0" ]]; then
- KSM_NODES=$(hostname)
-fi
-
-hadoop_uservar_su hdfs ksm "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${KSM_NODES}" \
- --daemon stop \
- ksm
-
-#---------------------------------------------------------
-# Ozone storagecontainermanager nodes
-SCM_NODES=$("${HADOOP_HDFS_HOME}/bin/hdfs" getconf -storagecontainermanagers 2>/dev/null)
-echo "Stopping storage container manager nodes [${SCM_NODES}]"
-hadoop_uservar_su hdfs scm "${HADOOP_HDFS_HOME}/bin/hdfs" \
- --workers \
- --config "${HADOOP_CONF_DIR}" \
- --hostnames "${SCM_NODES}" \
- --daemon stop \
- scm
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
deleted file mode 100644
index 77dd7ef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/conf/ozone-site.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
deleted file mode 100644
index 62e89f0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/CBlockManager.java
+++ /dev/null
@@ -1,421 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.BlockingService;
-import org.apache.hadoop.cblock.kubernetes.DynamicProvisioner;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.cblock.protocolPB
- .CBlockClientServerProtocolServerSideTranslatorPB;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.cblock.protocolPB
- .CBlockServiceProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.client.ContainerOperationClient;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.cblock.storage.StorageManager;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.protocolPB
- .StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.utils.LevelDBStore;
-import org.iq80.leveldb.DBIterator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SCM_IPADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SCM_IPADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SCM_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SCM_PORT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT;
-
-
-/**
- * The main entry point of CBlock operations, ALL the CBlock operations
- * will go through this class. But NOTE that:
- *
- * volume operations (create/
- * delete/info) are:
- * client -> CBlockManager -> StorageManager -> CBlock client
- *
- * IO operations (put/get block) are;
- * client -> CBlock client -> container
- *
- */
-public class CBlockManager implements CBlockServiceProtocol,
- CBlockClientProtocol {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CBlockManager.class);
-
- private final RPC.Server cblockService;
- private final RPC.Server cblockServer;
-
- private final StorageManager storageManager;
-
- private final LevelDBStore levelDBStore;
- private final String dbPath;
-
- private final DynamicProvisioner kubernetesDynamicProvisioner;
-
- private Charset encoding = Charset.forName("UTF-8");
-
- public CBlockManager(OzoneConfiguration conf,
- ScmClient storageClient) throws IOException {
- // Fix the cBlockManagerId generattion code here. Should support
- // cBlockManager --init command which will generate a cBlockManagerId and
- // persist it locally.
- storageManager =
- new StorageManager(storageClient, conf, "CBLOCK");
-
- dbPath = conf.getTrimmed(DFS_CBLOCK_SERVICE_LEVELDB_PATH_KEY,
- DFS_CBLOCK_SERVICE_LEVELDB_PATH_DEFAULT);
- levelDBStore = new LevelDBStore(new File(dbPath), true);
- LOG.info("Try to load exising volume information");
- readFromPersistentStore();
-
- RPC.setProtocolEngine(conf, CBlockServiceProtocolPB.class,
- ProtobufRpcEngine.class);
- RPC.setProtocolEngine(conf, CBlockClientServerProtocolPB.class,
- ProtobufRpcEngine.class);
- // start service for client command-to-cblock server service
- InetSocketAddress serviceRpcAddr =
- OzoneClientUtils.getCblockServiceRpcAddr(conf);
- BlockingService cblockProto =
- CBlockServiceProtocolProtos
- .CBlockServiceProtocolService
- .newReflectiveBlockingService(
- new CBlockServiceProtocolServerSideTranslatorPB(this)
- );
- cblockService = startRpcServer(conf, CBlockServiceProtocolPB.class,
- cblockProto, serviceRpcAddr,
- DFS_CBLOCK_SERVICERPC_BIND_HOST_KEY,
- DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
- DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
- InetSocketAddress cblockServiceRpcAddress =
- OzoneClientUtils.updateRPCListenAddress(conf,
- DFS_CBLOCK_SERVICERPC_ADDRESS_KEY, serviceRpcAddr, cblockService);
- LOG.info("CBlock manager listening for client commands on: {}",
- cblockServiceRpcAddress);
- // now start service for cblock client-to-cblock server communication
-
- InetSocketAddress serverRpcAddr =
- OzoneClientUtils.getCblockServerRpcAddr(conf);
- BlockingService serverProto =
- CBlockClientServerProtocolProtos
- .CBlockClientServerProtocolService
- .newReflectiveBlockingService(
- new CBlockClientServerProtocolServerSideTranslatorPB(this)
- );
- cblockServer = startRpcServer(
- conf, CBlockClientServerProtocolPB.class,
- serverProto, serverRpcAddr,
- DFS_CBLOCK_JSCSIRPC_BIND_HOST_KEY,
- DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_KEY,
- DFS_CBLOCK_SERVICERPC_HANDLER_COUNT_DEFAULT);
- InetSocketAddress cblockServerRpcAddress =
- OzoneClientUtils.updateRPCListenAddress(conf,
- DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY, serverRpcAddr, cblockServer);
- LOG.info("CBlock server listening for client commands on: {}",
- cblockServerRpcAddress);
-
- if (conf.getBoolean(DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED,
- DFS_CBLOCK_KUBERNETES_DYNAMIC_PROVISIONER_ENABLED_DEFAULT)) {
-
- kubernetesDynamicProvisioner =
- new DynamicProvisioner(conf, storageManager);
- kubernetesDynamicProvisioner.init();
-
- } else {
- kubernetesDynamicProvisioner = null;
- }
- }
-
- public void start() {
- cblockService.start();
- cblockServer.start();
- if (kubernetesDynamicProvisioner != null) {
- kubernetesDynamicProvisioner.start();
- }
- LOG.info("CBlock manager started!");
- }
-
- public void stop() {
- cblockService.stop();
- cblockServer.stop();
- if (kubernetesDynamicProvisioner != null) {
- kubernetesDynamicProvisioner.stop();
- }
- }
-
- public void join() {
- try {
- cblockService.join();
- cblockServer.join();
- } catch (InterruptedException e) {
- LOG.error("Interrupted during join");
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Starts an RPC server, if configured.
- *
- * @param conf configuration
- * @param protocol RPC protocol provided by RPC server
- * @param instance RPC protocol implementation instance
- * @param addr configured address of RPC server
- * @param bindHostKey configuration key for setting explicit bind host. If
- * the property is not configured, then the bind host is taken from addr.
- * @param handlerCountKey configuration key for RPC server handler count
- * @param handlerCountDefault default RPC server handler count if unconfigured
- * @return RPC server, or null if addr is null
- * @throws IOException if there is an I/O error while creating RPC server
- */
- private static RPC.Server startRpcServer(OzoneConfiguration conf,
- Class<?> protocol, BlockingService instance,
- InetSocketAddress addr, String bindHostKey,
- String handlerCountKey, int handlerCountDefault) throws IOException {
- if (addr == null) {
- return null;
- }
- String bindHost = conf.getTrimmed(bindHostKey);
- if (bindHost == null || bindHost.isEmpty()) {
- bindHost = addr.getHostName();
- }
- int numHandlers = conf.getInt(handlerCountKey, handlerCountDefault);
- RPC.Server rpcServer = new RPC.Builder(conf)
- .setProtocol(protocol)
- .setInstance(instance)
- .setBindAddress(bindHost)
- .setPort(addr.getPort())
- .setNumHandlers(numHandlers)
- .setVerbose(false)
- .setSecretManager(null)
- .build();
- return rpcServer;
- }
-
- @Override
- public synchronized MountVolumeResponse mountVolume(
- String userName, String volumeName) throws IOException {
- return storageManager.isVolumeValid(userName, volumeName);
- }
-
- @Override
- public List<VolumeInfo> listVolumes() throws IOException {
- return listVolume(null);
- }
-
- @Override
- public synchronized void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws IOException {
- LOG.info("Create volume received: userName: {} volumeName: {} " +
- "volumeSize: {} blockSize: {}", userName, volumeName,
- volumeSize, blockSize);
- // It is important to create in-memory representation of the
- // volume first, then writes to persistent storage (levelDB)
- // such that it is guaranteed that when there is an entry in
- // levelDB, the volume is allocated. (more like a UNDO log fashion)
- // TODO: what if creation failed? we allocated containers but lost
- // the reference to the volume and all it's containers. How to release
- // the containers?
- storageManager.createVolume(userName, volumeName, volumeSize, blockSize);
- VolumeDescriptor volume = storageManager.getVolume(userName, volumeName);
- if (volume == null) {
- throw new IOException("Volume creation failed!");
- }
- String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
- writeToPersistentStore(volumeKey.getBytes(encoding),
- volume.toProtobuf().toByteArray());
- }
-
- @Override
- public synchronized void deleteVolume(String userName,
- String volumeName, boolean force) throws IOException {
- LOG.info("Delete volume received: volume: {} {} ", volumeName, force);
- storageManager.deleteVolume(userName, volumeName, force);
- // being here means volume is successfully deleted now
- String volumeKey = KeyUtil.getVolumeKey(userName, volumeName);
- removeFromPersistentStore(volumeKey.getBytes(encoding));
- }
-
- // No need to synchronize on the following three methods, since write and
- // remove's caller are synchronized. read's caller is the constructor and
- // no other method call can happen at that time.
- @VisibleForTesting
- public void writeToPersistentStore(byte[] key, byte[] value) {
- levelDBStore.put(key, value);
- }
-
- @VisibleForTesting
- public void removeFromPersistentStore(byte[] key) {
- levelDBStore.delete(key);
- }
-
- public void readFromPersistentStore() throws IOException {
- try (DBIterator iter = levelDBStore.getIterator()) {
- iter.seekToFirst();
- while (iter.hasNext()) {
- Map.Entry<byte[], byte[]> entry = iter.next();
- String volumeKey = new String(entry.getKey(), encoding);
- try {
- VolumeDescriptor volumeDescriptor =
- VolumeDescriptor.fromProtobuf(entry.getValue());
- storageManager.addVolume(volumeDescriptor);
- } catch (IOException e) {
- LOG.error("Loading volume " + volumeKey + " error " + e);
- }
- }
- }
- }
-
- @Override
- public synchronized VolumeInfo infoVolume(String userName, String volumeName
- ) throws IOException {
- LOG.info("Info volume received: volume: {}", volumeName);
- return storageManager.infoVolume(userName, volumeName);
- }
-
- @VisibleForTesting
- public synchronized List<VolumeDescriptor> getAllVolumes() {
- return storageManager.getAllVolume(null);
- }
-
- public synchronized List<VolumeDescriptor> getAllVolumes(String userName) {
- return storageManager.getAllVolume(userName);
- }
-
- public synchronized void close() {
- try {
- levelDBStore.close();
- } catch (IOException e) {
- LOG.error("Error when closing levelDB " + e);
- }
- }
-
- public synchronized void clean() {
- try {
- levelDBStore.close();
- levelDBStore.destroy();
- } catch (IOException e) {
- LOG.error("Error when deleting levelDB " + e);
- }
- }
-
- @Override
- public synchronized List<VolumeInfo> listVolume(String userName)
- throws IOException {
- ArrayList<VolumeInfo> response = new ArrayList<>();
- List<VolumeDescriptor> allVolumes =
- storageManager.getAllVolume(userName);
- for (VolumeDescriptor volume : allVolumes) {
- VolumeInfo info =
- new VolumeInfo(volume.getUserName(), volume.getVolumeName(),
- volume.getVolumeSize(), volume.getBlockSize());
- response.add(info);
- }
- return response;
- }
-
- public static void main(String[] args) throws Exception {
- long version = RPC.getProtocolVersion(
- StorageContainerLocationProtocolPB.class);
- OzoneConfiguration ozoneConf = new OzoneConfiguration();
- String scmAddress = ozoneConf.get(DFS_CBLOCK_SCM_IPADDRESS_KEY,
- DFS_CBLOCK_SCM_IPADDRESS_DEFAULT);
- int scmPort = ozoneConf.getInt(DFS_CBLOCK_SCM_PORT_KEY,
- DFS_CBLOCK_SCM_PORT_DEFAULT);
- int containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
- DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
- ContainerOperationClient.setContainerSizeB(containerSizeGB* OzoneConsts.GB);
- InetSocketAddress address = new InetSocketAddress(scmAddress, scmPort);
-
- ozoneConf.set(OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT,
- OzoneConfigKeys.OZONE_LOCALSTORAGE_ROOT_DEFAULT);
- LOG.info(
- "Creating StorageContainerLocationProtocol RPC client with address {}",
- address);
- RPC.setProtocolEngine(ozoneConf, StorageContainerLocationProtocolPB.class,
- ProtobufRpcEngine.class);
- StorageContainerLocationProtocolClientSideTranslatorPB client =
- new StorageContainerLocationProtocolClientSideTranslatorPB(
- RPC.getProxy(StorageContainerLocationProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), ozoneConf,
- NetUtils.getDefaultSocketFactory(ozoneConf),
- Client.getRpcTimeout(ozoneConf)));
- ScmClient storageClient = new ContainerOperationClient(
- client, new XceiverClientManager(ozoneConf));
- CBlockManager cbm = new CBlockManager(ozoneConf, storageClient);
- cbm.start();
- cbm.join();
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
deleted file mode 100644
index 23ba00b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerMetrics.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableQuantiles;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-
-/**
- *
- * This class is for maintaining the various Storage Container
- * DataNode statistics and publishing them through the metrics interfaces.
- * This also registers the JMX MBean for RPC.
- * <p>
- * This class has a number of metrics variables that are publicly accessible;
- * these variables (objects) have methods to update their values;
- * for example:
- * <p> {@link #numOps}.inc()
- *
- */
-@InterfaceAudience.Private
-@Metrics(about="Storage Container DataNode Metrics", context="dfs")
-public class ContainerMetrics {
- @Metric private MutableCounterLong numOps;
- private MutableCounterLong[] numOpsArray;
- private MutableCounterLong[] opsBytesArray;
- private MutableRate[] opsLatency;
- private MutableQuantiles[][] opsLatQuantiles;
- private MetricsRegistry registry = null;
-
- public ContainerMetrics(int[] intervals) {
- int numEnumEntries = ContainerProtos.Type.values().length;
- final int len = intervals.length;
- this.numOpsArray = new MutableCounterLong[numEnumEntries];
- this.opsBytesArray = new MutableCounterLong[numEnumEntries];
- this.opsLatency = new MutableRate[numEnumEntries];
- this.opsLatQuantiles = new MutableQuantiles[numEnumEntries][len];
- this.registry = new MetricsRegistry("StorageContainerMetrics");
- for (int i = 0; i < numEnumEntries; i++) {
- numOpsArray[i] = registry.newCounter(
- "num" + ContainerProtos.Type.valueOf(i + 1),
- "number of " + ContainerProtos.Type.valueOf(i + 1) + " ops",
- (long) 0);
- opsBytesArray[i] = registry.newCounter(
- "bytes" + ContainerProtos.Type.valueOf(i + 1),
- "bytes used by " + ContainerProtos.Type.valueOf(i + 1) + "op",
- (long) 0);
- opsLatency[i] = registry.newRate(
- "latency" + ContainerProtos.Type.valueOf(i + 1),
- ContainerProtos.Type.valueOf(i + 1) + " op");
-
- for (int j = 0; j < len; j++) {
- int interval = intervals[j];
- String quantileName = ContainerProtos.Type.valueOf(i + 1) + "Nanos"
- + interval + "s";
- opsLatQuantiles[i][j] = registry.newQuantiles(quantileName,
- "latency of Container ops", "ops", "latency", interval);
- }
- }
- }
-
- public static ContainerMetrics create(Configuration conf) {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- // Percentile measurement is off by default, by watching no intervals
- int[] intervals =
- conf.getInts(DFSConfigKeys.DFS_METRICS_PERCENTILES_INTERVALS_KEY);
- return ms.register("StorageContainerMetrics",
- "Storage Container Node Metrics",
- new ContainerMetrics(intervals));
- }
-
- public void incContainerOpcMetrics(ContainerProtos.Type type){
- numOps.incr();
- numOpsArray[type.ordinal()].incr();
- }
-
- public long getContainerOpsMetrics(ContainerProtos.Type type){
- return numOpsArray[type.ordinal()].value();
- }
-
- public void incContainerOpsLatencies(ContainerProtos.Type type,
- long latencyNanos) {
- opsLatency[type.ordinal()].add(latencyNanos);
- for (MutableQuantiles q: opsLatQuantiles[type.ordinal()]) {
- q.add(latencyNanos);
- }
- }
-
- public void incContainerBytesStats(ContainerProtos.Type type, long bytes) {
- opsBytesArray[type.ordinal()].incr(bytes);
- }
-
- public long getContainerBytesMetrics(ContainerProtos.Type type){
- return opsBytesArray[type.ordinal()].value();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
deleted file mode 100644
index 81c41bb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerReport.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerInfo;
-
-/**
- * Container Report iterates the closed containers and sends a container report
- * to SCM.
- */
-public class ContainerReport {
- private static final int UNKNOWN = -1;
- private final String containerName;
- private final String finalhash;
- private long size;
- private long keyCount;
- private long bytesUsed;
- private long readCount;
- private long writeCount;
- private long readBytes;
- private long writeBytes;
- private long containerID;
-
- public long getContainerID() {
- return containerID;
- }
-
- public void setContainerID(long containerID) {
- this.containerID = containerID;
- }
-
-
-
-
- /**
- * Constructs the ContainerReport.
- *
- * @param containerName - Container Name.
- * @param finalhash - Final Hash.
- */
- public ContainerReport(String containerName, String finalhash) {
- this.containerName = containerName;
- this.finalhash = finalhash;
- this.size = UNKNOWN;
- this.keyCount = UNKNOWN;
- this.bytesUsed = 0L;
- this.readCount = 0L;
- this.readBytes = 0L;
- this.writeCount = 0L;
- this.writeBytes = 0L;
- }
-
- /**
- * Gets a containerReport from protobuf class.
- *
- * @param info - ContainerInfo.
- * @return - ContainerReport.
- */
- public static ContainerReport getFromProtoBuf(ContainerInfo info) {
- Preconditions.checkNotNull(info);
- ContainerReport report = new ContainerReport(info.getContainerName(),
- info.getFinalhash());
- if (info.hasSize()) {
- report.setSize(info.getSize());
- }
- if (info.hasKeyCount()) {
- report.setKeyCount(info.getKeyCount());
- }
- if (info.hasUsed()) {
- report.setBytesUsed(info.getUsed());
- }
- if (info.hasReadCount()) {
- report.setReadCount(info.getReadCount());
- }
- if (info.hasReadBytes()) {
- report.setReadBytes(info.getReadBytes());
- }
- if (info.hasWriteCount()) {
- report.setWriteCount(info.getWriteCount());
- }
- if (info.hasWriteBytes()) {
- report.setWriteBytes(info.getWriteBytes());
- }
-
- report.setContainerID(info.getContainerID());
- return report;
- }
-
- /**
- * Gets the container name.
- *
- * @return - Name
- */
- public String getContainerName() {
- return containerName;
- }
-
- /**
- * Returns the final signature for this container.
- *
- * @return - hash
- */
- public String getFinalhash() {
- return finalhash;
- }
-
- /**
- * Returns a positive number it is a valid number, -1 if not known.
- *
- * @return size or -1
- */
- public long getSize() {
- return size;
- }
-
- /**
- * Sets the size of the container on disk.
- *
- * @param size - int
- */
- public void setSize(long size) {
- this.size = size;
- }
-
- /**
- * Gets number of keys in the container if known.
- *
- * @return - Number of keys or -1 for not known.
- */
- public long getKeyCount() {
- return keyCount;
- }
-
- /**
- * Sets the key count.
- *
- * @param keyCount - Key Count
- */
- public void setKeyCount(long keyCount) {
- this.keyCount = keyCount;
- }
-
- public long getReadCount() {
- return readCount;
- }
-
- public void setReadCount(long readCount) {
- this.readCount = readCount;
- }
-
- public long getWriteCount() {
- return writeCount;
- }
-
- public void setWriteCount(long writeCount) {
- this.writeCount = writeCount;
- }
-
- public long getReadBytes() {
- return readBytes;
- }
-
- public void setReadBytes(long readBytes) {
- this.readBytes = readBytes;
- }
-
- public long getWriteBytes() {
- return writeBytes;
- }
-
- public void setWriteBytes(long writeBytes) {
- this.writeBytes = writeBytes;
- }
-
- public long getBytesUsed() {
- return bytesUsed;
- }
-
- public void setBytesUsed(long bytesUsed) {
- this.bytesUsed = bytesUsed;
- }
-
- /**
- * Gets a containerInfo protobuf message from ContainerReports.
- *
- * @return ContainerInfo
- */
- public ContainerInfo getProtoBufMessage() {
- return ContainerInfo.newBuilder()
- .setContainerName(this.getContainerName())
- .setKeyCount(this.getKeyCount())
- .setSize(this.getSize())
- .setUsed(this.getBytesUsed())
- .setReadCount(this.getReadCount())
- .setReadBytes(this.getReadBytes())
- .setWriteCount(this.getWriteCount())
- .setWriteBytes(this.getWriteBytes())
- .setFinalhash(this.getFinalhash())
- .setContainerID(this.getContainerID())
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
deleted file mode 100644
index f78e646..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java
+++ /dev/null
@@ -1,477 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.commons.io.FilenameUtils.removeExtension;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.INVALID_ARGUMENT;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNABLE_TO_FIND_DATA_DIR;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_EXTENSION;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_META;
-
-/**
- * A set of helper functions to create proper responses.
- */
-public final class ContainerUtils {
-
- private ContainerUtils() {
- //never constructed.
- }
-
- /**
- * Returns a CreateContainer Response. This call is used by create and delete
- * containers which have null success responses.
- *
- * @param msg Request
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getContainerResponse(ContainerProtos.ContainerCommandRequestProto msg) {
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- getContainerResponse(msg, ContainerProtos.Result.SUCCESS, "");
- return builder.build();
- }
-
- /**
- * Returns a ReadContainer Response.
- *
- * @param msg Request
- * @param containerData - data
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getReadContainerResponse(ContainerProtos.ContainerCommandRequestProto msg,
- ContainerData containerData) {
- Preconditions.checkNotNull(containerData);
-
- ContainerProtos.ReadContainerResponseProto.Builder response =
- ContainerProtos.ReadContainerResponseProto.newBuilder();
- response.setContainerData(containerData.getProtoBufMessage());
-
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- getContainerResponse(msg, ContainerProtos.Result.SUCCESS, "");
- builder.setReadContainer(response);
- return builder.build();
- }
-
- /**
- * We found a command type but no associated payload for the command. Hence
- * return malformed Command as response.
- *
- * @param msg - Protobuf message.
- * @param result - result
- * @param message - Error message.
- * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
- */
- public static ContainerProtos.ContainerCommandResponseProto.Builder
- getContainerResponse(ContainerProtos.ContainerCommandRequestProto msg,
- ContainerProtos.Result result, String message) {
- return
- ContainerProtos.ContainerCommandResponseProto.newBuilder()
- .setCmdType(msg.getCmdType())
- .setTraceID(msg.getTraceID())
- .setResult(result)
- .setMessage(message);
- }
-
- /**
- * Logs the error and returns a response to the caller.
- *
- * @param log - Logger
- * @param ex - Exception
- * @param msg - Request Object
- * @return Response
- */
- public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
- Logger log, StorageContainerException ex,
- ContainerProtos.ContainerCommandRequestProto msg) {
- log.info("Operation: {} : Trace ID: {} : Message: {} : Result: {}",
- msg.getCmdType().name(), msg.getTraceID(),
- ex.getMessage(), ex.getResult().getValueDescriptor().getName());
- return getContainerResponse(msg, ex.getResult(), ex.getMessage()).build();
- }
-
- /**
- * Logs the error and returns a response to the caller.
- *
- * @param log - Logger
- * @param ex - Exception
- * @param msg - Request Object
- * @return Response
- */
- public static ContainerProtos.ContainerCommandResponseProto logAndReturnError(
- Logger log, RuntimeException ex,
- ContainerProtos.ContainerCommandRequestProto msg) {
- log.info("Operation: {} : Trace ID: {} : Message: {} ",
- msg.getCmdType().name(), msg.getTraceID(), ex.getMessage());
- return getContainerResponse(msg, INVALID_ARGUMENT, ex.getMessage()).build();
- }
-
- /**
- * We found a command type but no associated payload for the command. Hence
- * return malformed Command as response.
- *
- * @param msg - Protobuf message.
- * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- malformedRequest(ContainerProtos.ContainerCommandRequestProto msg) {
- return getContainerResponse(msg, ContainerProtos.Result.MALFORMED_REQUEST,
- "Cmd type does not match the payload.").build();
- }
-
- /**
- * We found a command type that is not supported yet.
- *
- * @param msg - Protobuf message.
- * @return ContainerCommandResponseProto - MALFORMED_REQUEST.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- unsupportedRequest(ContainerProtos.ContainerCommandRequestProto msg) {
- return getContainerResponse(msg, ContainerProtos.Result.UNSUPPORTED_REQUEST,
- "Server does not support this command yet.").build();
- }
-
- /**
- * get containerName from a container file.
- *
- * @param containerFile - File
- * @return Name of the container.
- */
- public static String getContainerNameFromFile(File containerFile) {
- Preconditions.checkNotNull(containerFile);
- return Paths.get(containerFile.getParent()).resolve(
- removeExtension(containerFile.getName())).toString();
- }
-
- /**
- * Verifies that this in indeed a new container.
- *
- * @param containerFile - Container File to verify
- * @param metadataFile - metadata File to verify
- * @throws IOException
- */
- public static void verifyIsNewContainer(File containerFile, File metadataFile)
- throws IOException {
- Logger log = LoggerFactory.getLogger(ContainerManagerImpl.class);
- if (containerFile.exists()) {
- log.error("container already exists on disk. File: {}",
- containerFile.toPath());
- throw new FileAlreadyExistsException("container already exists on " +
- "disk.");
- }
-
- if (metadataFile.exists()) {
- log.error("metadata found on disk, but missing container. Refusing to" +
- " write this container. File: {} ", metadataFile.toPath());
- throw new FileAlreadyExistsException(("metadata found on disk, but " +
- "missing container. Refusing to write this container."));
- }
-
- File parentPath = new File(containerFile.getParent());
-
- if (!parentPath.exists() && !parentPath.mkdirs()) {
- log.error("Unable to create parent path. Path: {}",
- parentPath.toString());
- throw new IOException("Unable to create container directory.");
- }
-
- if (!containerFile.createNewFile()) {
- log.error("creation of a new container file failed. File: {}",
- containerFile.toPath());
- throw new IOException("creation of a new container file failed.");
- }
-
- if (!metadataFile.createNewFile()) {
- log.error("creation of the metadata file failed. File: {}",
- metadataFile.toPath());
- throw new IOException("creation of a new container file failed.");
- }
- }
-
- public static String getContainerDbFileName(String containerName) {
- return containerName + OzoneConsts.DN_CONTAINER_DB;
- }
-
- /**
- * creates a Metadata DB for the specified container.
- *
- * @param containerPath - Container Path.
- * @throws IOException
- */
- public static Path createMetadata(Path containerPath, String containerName,
- Configuration conf)
- throws IOException {
- Logger log = LoggerFactory.getLogger(ContainerManagerImpl.class);
- Preconditions.checkNotNull(containerPath);
- Path metadataPath = containerPath.resolve(OzoneConsts.CONTAINER_META_PATH);
- if (!metadataPath.toFile().mkdirs()) {
- log.error("Unable to create directory for metadata storage. Path: {}",
- metadataPath);
- throw new IOException("Unable to create directory for metadata storage." +
- " Path: " + metadataPath);
- }
- MetadataStore store = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setCreateIfMissing(true)
- .setDbFile(metadataPath
- .resolve(getContainerDbFileName(containerName)).toFile())
- .build();
-
- // we close since the SCM pre-creates containers.
- // we will open and put Db handle into a cache when keys are being created
- // in a container.
-
- store.close();
-
- Path dataPath = containerPath.resolve(OzoneConsts.CONTAINER_DATA_PATH);
- if (!dataPath.toFile().mkdirs()) {
-
- // If we failed to create data directory, we cleanup the
- // metadata directory completely. That is, we will delete the
- // whole directory including LevelDB file.
- log.error("Unable to create directory for data storage. cleaning up the" +
- " container path: {} dataPath: {}",
- containerPath, dataPath);
- FileUtils.deleteDirectory(containerPath.toFile());
- throw new IOException("Unable to create directory for data storage." +
- " Path: " + dataPath);
- }
- return metadataPath;
- }
-
- /**
- * Returns Metadata location.
- *
- * @param containerData - Data
- * @param location - Path
- * @return Path
- */
- public static File getMetadataFile(ContainerData containerData,
- Path location) {
- return location.resolve(containerData
- .getContainerName().concat(CONTAINER_META))
- .toFile();
- }
-
- /**
- * Returns container file location.
- *
- * @param containerData - Data
- * @param location - Root path
- * @return Path
- */
- public static File getContainerFile(ContainerData containerData,
- Path location) {
- return location.resolve(containerData
- .getContainerName().concat(CONTAINER_EXTENSION))
- .toFile();
- }
-
- /**
- * Container metadata directory -- here is where the level DB lives.
- *
- * @param cData - cData.
- * @return Path to the parent directory where the DB lives.
- */
- public static Path getMetadataDirectory(ContainerData cData) {
- Path dbPath = Paths.get(cData.getDBPath());
- Preconditions.checkNotNull(dbPath);
- Preconditions.checkState(dbPath.toString().length() > 0);
- return dbPath.getParent();
- }
-
- /**
- * Returns the path where data or chunks live for a given container.
- *
- * @param cData - cData container
- * @return - Path
- * @throws StorageContainerException
- */
- public static Path getDataDirectory(ContainerData cData)
- throws StorageContainerException {
- Path path = getMetadataDirectory(cData);
- Preconditions.checkNotNull(path);
- Path parentPath = path.getParent();
- if (parentPath == null) {
- throw new StorageContainerException("Unable to get Data directory."
- + path, UNABLE_TO_FIND_DATA_DIR);
- }
- return parentPath.resolve(OzoneConsts.CONTAINER_DATA_PATH);
- }
-
- /**
- * remove Container if it is empty.
- * <p/>
- * There are three things we need to delete.
- * <p/>
- * 1. Container file and metadata file. 2. The Level DB file 3. The path that
- * we created on the data location.
- *
- * @param containerData - Data of the container to remove.
- * @param conf - configuration of the cluster.
- * @param forceDelete - whether this container should be deleted forcibly.
- * @throws IOException
- */
- public static void removeContainer(ContainerData containerData,
- Configuration conf, boolean forceDelete) throws IOException {
- Preconditions.checkNotNull(containerData);
- Path dbPath = Paths.get(containerData.getDBPath());
-
- MetadataStore db = KeyUtils.getDB(containerData, conf);
- // If the container is not empty and cannot be deleted forcibly,
- // then throw a SCE to stop deleting.
- if(!forceDelete && !db.isEmpty()) {
- throw new StorageContainerException(
- "Container cannot be deleted because it is not empty.",
- ContainerProtos.Result.ERROR_CONTAINER_NOT_EMPTY);
- }
- // Close the DB connection and remove the DB handler from cache
- KeyUtils.removeDB(containerData, conf);
-
- // Delete the DB File.
- FileUtils.forceDelete(dbPath.toFile());
- dbPath = dbPath.getParent();
-
- // Delete all Metadata in the Data directories for this containers.
- if (dbPath != null) {
- FileUtils.deleteDirectory(dbPath.toFile());
- dbPath = dbPath.getParent();
- }
-
- // now delete the container directory, this means that all key data dirs
- // will be removed too.
- if (dbPath != null) {
- FileUtils.deleteDirectory(dbPath.toFile());
- }
-
- // Delete the container metadata from the metadata locations.
- String rootPath = getContainerNameFromFile(new File(containerData
- .getContainerPath()));
- Path containerPath = Paths.get(rootPath.concat(CONTAINER_EXTENSION));
- Path metaPath = Paths.get(rootPath.concat(CONTAINER_META));
-
- FileUtils.forceDelete(containerPath.toFile());
- FileUtils.forceDelete(metaPath.toFile());
- }
-
- /**
- * Write datanode ID protobuf messages to an ID file.
- * The old ID file will be overwritten.
- *
- * @param ids A set of {@link DatanodeID}
- * @param path Local ID file path
- * @throws IOException When read/write error occurs
- */
- private synchronized static void writeDatanodeIDs(List<DatanodeID> ids,
- File path) throws IOException {
- if (path.exists()) {
- if (!path.delete() || !path.createNewFile()) {
- throw new IOException("Unable to overwrite the datanode ID file.");
- }
- } else {
- if(!path.getParentFile().exists() &&
- !path.getParentFile().mkdirs()) {
- throw new IOException("Unable to create datanode ID directories.");
- }
- }
- try (FileOutputStream out = new FileOutputStream(path)) {
- for (DatanodeID id : ids) {
- HdfsProtos.DatanodeIDProto dnId = id.getProtoBufMessage();
- dnId.writeDelimitedTo(out);
- }
- }
- }
-
- /**
- * Persistent a {@link DatanodeID} to a local file.
- * It reads the IDs first and append a new entry only if the ID is new.
- * This is to avoid on some dirty environment, this file gets too big.
- *
- * @throws IOException when read/write error occurs
- */
- public synchronized static void writeDatanodeIDTo(DatanodeID dnID,
- File path) throws IOException {
- List<DatanodeID> ids = ContainerUtils.readDatanodeIDsFrom(path);
- // Only create or overwrite the file
- // if the ID doesn't exist in the ID file
- for (DatanodeID id : ids) {
- if (id.getProtoBufMessage()
- .equals(dnID.getProtoBufMessage())) {
- return;
- }
- }
- ids.add(dnID);
- writeDatanodeIDs(ids, path);
- }
-
- /**
- * Read {@link DatanodeID} from a local ID file and return a set of
- * datanode IDs. If the ID file doesn't exist, an empty set is returned.
- *
- * @param path ID file local path
- * @return A set of {@link DatanodeID}
- * @throws IOException If the id file is malformed or other I/O exceptions
- */
- public synchronized static List<DatanodeID> readDatanodeIDsFrom(File path)
- throws IOException {
- List<DatanodeID> ids = new ArrayList<DatanodeID>();
- if (!path.exists()) {
- return ids;
- }
- try(FileInputStream in = new FileInputStream(path)) {
- while(in.available() > 0) {
- try {
- HdfsProtos.DatanodeIDProto id =
- HdfsProtos.DatanodeIDProto.parseDelimitedFrom(in);
- ids.add(DatanodeID.getFromProtoBuf(id));
- } catch (IOException e) {
- throw new IOException("Failed to parse Datanode ID from "
- + path.getAbsolutePath(), e);
- }
- }
- }
- return ids;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
deleted file mode 100644
index de5e2d0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/DeletedContainerBlocksSummary.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.collect.Maps;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
-
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/**
- * A helper class to wrap the info about under deletion container blocks.
- */
-public final class DeletedContainerBlocksSummary {
-
- private final List<DeletedBlocksTransaction> blocks;
- // key : txID
- // value : times of this tx has been processed
- private final Map<Long, Integer> txSummary;
- // key : container name
- // value : the number of blocks need to be deleted in this container
- // if the message contains multiple entries for same block,
- // blocks will be merged
- private final Map<String, Integer> blockSummary;
- // total number of blocks in this message
- private int numOfBlocks;
-
- private DeletedContainerBlocksSummary(List<DeletedBlocksTransaction> blocks) {
- this.blocks = blocks;
- txSummary = Maps.newHashMap();
- blockSummary = Maps.newHashMap();
- blocks.forEach(entry -> {
- txSummary.put(entry.getTxID(), entry.getCount());
- if (blockSummary.containsKey(entry.getContainerName())) {
- blockSummary.put(entry.getContainerName(),
- blockSummary.get(entry.getContainerName())
- + entry.getBlockIDCount());
- } else {
- blockSummary.put(entry.getContainerName(), entry.getBlockIDCount());
- }
- numOfBlocks += entry.getBlockIDCount();
- });
- }
-
- public static DeletedContainerBlocksSummary getFrom(
- List<DeletedBlocksTransaction> blocks) {
- return new DeletedContainerBlocksSummary(blocks);
- }
-
- public int getNumOfBlocks() {
- return numOfBlocks;
- }
-
- public int getNumOfContainers() {
- return blockSummary.size();
- }
-
- public String getTXIDs() {
- return String.join(",", txSummary.keySet()
- .stream().map(String::valueOf).collect(Collectors.toList()));
- }
-
- public String getTxIDSummary() {
- List<String> txSummaryEntry = txSummary.entrySet().stream()
- .map(entry -> entry.getKey() + "(" + entry.getValue() + ")")
- .collect(Collectors.toList());
- return "[" + String.join(",", txSummaryEntry) + "]";
- }
-
- @Override public String toString() {
- StringBuffer sb = new StringBuffer();
- for (DeletedBlocksTransaction blks : blocks) {
- sb.append(" ")
- .append("TXID=")
- .append(blks.getTxID())
- .append(", ")
- .append("TimesProceed=")
- .append(blks.getCount())
- .append(", ")
- .append(blks.getContainerName())
- .append(" : [")
- .append(String.join(",", blks.getBlockIDList())).append("]")
- .append("\n");
- }
- return sb.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
deleted file mode 100644
index 27731c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/FileUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.ByteString;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-
-/**
- * File Utils are helper routines used by putSmallFile and getSmallFile
- * RPCs.
- */
-public final class FileUtils {
- /**
- * Never Constructed.
- */
- private FileUtils() {
- }
-
- /**
- * Gets a response for the putSmallFile RPC.
- * @param msg - ContainerCommandRequestProto
- * @return - ContainerCommandResponseProto
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getPutFileResponse(ContainerProtos.ContainerCommandRequestProto msg) {
- ContainerProtos.PutSmallFileResponseProto.Builder getResponse =
- ContainerProtos.PutSmallFileResponseProto.newBuilder();
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
- .SUCCESS, "");
- builder.setCmdType(ContainerProtos.Type.PutSmallFile);
- builder.setPutSmallFile(getResponse);
- return builder.build();
- }
-
- /**
- * Gets a response to the read small file call.
- * @param msg - Msg
- * @param data - Data
- * @param info - Info
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getGetSmallFileResponse(ContainerProtos.ContainerCommandRequestProto msg,
- byte[] data, ChunkInfo info) {
- Preconditions.checkNotNull(msg);
-
- ContainerProtos.ReadChunkResponseProto.Builder readChunkresponse =
- ContainerProtos.ReadChunkResponseProto.newBuilder();
- readChunkresponse.setChunkData(info.getProtoBufMessage());
- readChunkresponse.setData(ByteString.copyFrom(data));
- readChunkresponse.setPipeline(msg.getGetSmallFile().getKey().getPipeline());
-
- ContainerProtos.GetSmallFileResponseProto.Builder getSmallFile =
- ContainerProtos.GetSmallFileResponseProto.newBuilder();
- getSmallFile.setData(readChunkresponse.build());
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
- .SUCCESS, "");
- builder.setCmdType(ContainerProtos.Type.GetSmallFile);
- builder.setGetSmallFile(getSmallFile);
- return builder.build();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
deleted file mode 100644
index 639de16..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyData.java
+++ /dev/null
@@ -1,170 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-/**
- * Helper class to convert Protobuf to Java classes.
- */
-public class KeyData {
- private final String containerName;
- private final String keyName;
- private final Map<String, String> metadata;
-
- /**
- * Please note : when we are working with keys, we don't care what they point
- * to. So we We don't read chunkinfo nor validate them. It is responsibility
- * of higher layer like ozone. We just read and write data from network.
- */
- private List<ContainerProtos.ChunkInfo> chunks;
-
- /**
- * Constructs a KeyData Object.
- *
- * @param containerName
- * @param keyName
- */
- public KeyData(String containerName, String keyName) {
- this.containerName = containerName;
- this.keyName = keyName;
- this.metadata = new TreeMap<>();
- }
-
- /**
- * Returns a keyData object from the protobuf data.
- *
- * @param data - Protobuf data.
- * @return - KeyData
- * @throws IOException
- */
- public static KeyData getFromProtoBuf(ContainerProtos.KeyData data) throws
- IOException {
- KeyData keyData = new KeyData(data.getContainerName(), data.getName());
- for (int x = 0; x < data.getMetadataCount(); x++) {
- keyData.addMetadata(data.getMetadata(x).getKey(),
- data.getMetadata(x).getValue());
- }
- keyData.setChunks(data.getChunksList());
- return keyData;
- }
-
- /**
- * Returns a Protobuf message from KeyData.
- * @return Proto Buf Message.
- */
- public ContainerProtos.KeyData getProtoBufMessage() {
- ContainerProtos.KeyData.Builder builder =
- ContainerProtos.KeyData.newBuilder();
- builder.setContainerName(this.containerName);
- builder.setName(this.getKeyName());
- builder.addAllChunks(this.chunks);
- for (Map.Entry<String, String> entry : metadata.entrySet()) {
- OzoneProtos.KeyValue.Builder keyValBuilder =
- OzoneProtos.KeyValue.newBuilder();
- builder.addMetadata(keyValBuilder.setKey(entry.getKey())
- .setValue(entry.getValue()).build());
- }
- return builder.build();
- }
-
- /**
- * Adds metadata.
- *
- * @param key - Key
- * @param value - Value
- * @throws IOException
- */
- public synchronized void addMetadata(String key, String value) throws
- IOException {
- if (this.metadata.containsKey(key)) {
- throw new IOException("This key already exists. Key " + key);
- }
- metadata.put(key, value);
- }
-
- public synchronized Map<String, String> getMetadata() {
- return Collections.unmodifiableMap(this.metadata);
- }
-
- /**
- * Returns value of a key.
- */
- public synchronized String getValue(String key) {
- return metadata.get(key);
- }
-
- /**
- * Deletes a metadata entry from the map.
- *
- * @param key - Key
- */
- public synchronized void deleteKey(String key) {
- metadata.remove(key);
- }
-
- /**
- * Returns chunks list.
- *
- * @return list of chunkinfo.
- */
- public List<ContainerProtos.ChunkInfo> getChunks() {
- return chunks;
- }
-
- /**
- * Returns container Name.
- * @return String.
- */
- public String getContainerName() {
- return containerName;
- }
-
- /**
- * Returns KeyName.
- * @return String.
- */
- public String getKeyName() {
- return keyName;
- }
-
- /**
- * Sets Chunk list.
- *
- * @param chunks - List of chunks.
- */
- public void setChunks(List<ContainerProtos.ChunkInfo> chunks) {
- this.chunks = chunks;
- }
-
- /**
- * Get the total size of chunks allocated for the key.
- * @return total size of the key.
- */
- public long getSize() {
- return chunks.parallelStream().mapToLong(e->e.getLen()).sum();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
deleted file mode 100644
index b1c2b00..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/KeyUtils.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.utils.MetadataStore;
-
-import java.io.IOException;
-import java.nio.charset.Charset;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNABLE_TO_READ_METADATA_DB;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.NO_SUCH_KEY;
-
-/**
- * Utils functions to help key functions.
- */
-public final class KeyUtils {
- public static final String ENCODING_NAME = "UTF-8";
- public static final Charset ENCODING = Charset.forName(ENCODING_NAME);
-
- /**
- * Never Constructed.
- */
- private KeyUtils() {
- }
-
- /**
- * Get a DB handler for a given container.
- * If the handler doesn't exist in cache yet, first create one and
- * add into cache. This function is called with containerManager
- * ReadLock held.
- *
- * @param container container.
- * @param conf configuration.
- * @return MetadataStore handle.
- * @throws StorageContainerException
- */
- public static MetadataStore getDB(ContainerData container,
- Configuration conf) throws StorageContainerException {
- Preconditions.checkNotNull(container);
- ContainerCache cache = ContainerCache.getInstance(conf);
- Preconditions.checkNotNull(cache);
- try {
- return cache.getDB(container.getContainerName(), container.getDBPath());
- } catch (IOException ex) {
- String message =
- String.format("Unable to open DB. DB Name: %s, Path: %s. ex: %s",
- container.getContainerName(), container.getDBPath(), ex.getMessage());
- throw new StorageContainerException(message, UNABLE_TO_READ_METADATA_DB);
- }
- }
-
- /**
- * Remove a DB handler from cache.
- *
- * @param container - Container data.
- * @param conf - Configuration.
- */
- public static void removeDB(ContainerData container,
- Configuration conf) {
- Preconditions.checkNotNull(container);
- ContainerCache cache = ContainerCache.getInstance(conf);
- Preconditions.checkNotNull(cache);
- cache.removeDB(container.getContainerName());
- }
- /**
- * Shutdown all DB Handles.
- *
- * @param cache - Cache for DB Handles.
- */
- @SuppressWarnings("unchecked")
- public static void shutdownCache(ContainerCache cache) {
- cache.shutdownCache();
- }
-
- /**
- * Returns successful keyResponse.
- * @param msg - Request.
- * @return Response.
- */
- public static ContainerProtos.ContainerCommandResponseProto
- getKeyResponse(ContainerProtos.ContainerCommandRequestProto msg) {
- return ContainerUtils.getContainerResponse(msg);
- }
-
-
- public static ContainerProtos.ContainerCommandResponseProto
- getKeyDataResponse(ContainerProtos.ContainerCommandRequestProto msg,
- KeyData data) {
- ContainerProtos.GetKeyResponseProto.Builder getKey = ContainerProtos
- .GetKeyResponseProto.newBuilder();
- getKey.setKeyData(data.getProtoBufMessage());
- ContainerProtos.ContainerCommandResponseProto.Builder builder =
- ContainerUtils.getContainerResponse(msg, ContainerProtos.Result
- .SUCCESS, "");
- builder.setGetKey(getKey);
- return builder.build();
- }
-
- /**
- * Parses the key name from a bytes array.
- * @param bytes key name in bytes.
- * @return key name string.
- */
- public static String getKeyName(byte[] bytes) {
- return new String(bytes, ENCODING);
- }
-
- /**
- * Parses the {@link KeyData} from a bytes array.
- *
- * @param bytes key data in bytes.
- * @return key data.
- * @throws IOException if the bytes array is malformed or invalid.
- */
- public static KeyData getKeyData(byte[] bytes) throws IOException {
- try {
- ContainerProtos.KeyData kd = ContainerProtos.KeyData.parseFrom(bytes);
- KeyData data = KeyData.getFromProtoBuf(kd);
- return data;
- } catch (IOException e) {
- throw new StorageContainerException("Failed to parse key data from the" +
- " bytes array.", NO_SUCH_KEY);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
deleted file mode 100644
index 21f31e1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/helpers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.helpers;
-/**
- Contains protocol buffer helper classes and utilites used in
- impl.
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
deleted file mode 100644
index 9052df7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ChunkManagerImpl.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.file.Files;
-import java.nio.file.StandardCopyOption;
-import java.security.NoSuchAlgorithmException;
-import java.util.concurrent.ExecutionException;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.CONTAINER_INTERNAL_ERROR;
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.UNSUPPORTED_REQUEST;
-
-/**
- * An implementation of ChunkManager that is used by default in ozone.
- */
-public class ChunkManagerImpl implements ChunkManager {
- static final Logger LOG =
- LoggerFactory.getLogger(ChunkManagerImpl.class);
-
- private final ContainerManager containerManager;
-
- /**
- * Constructs a ChunkManager.
- *
- * @param manager - ContainerManager.
- */
- public ChunkManagerImpl(ContainerManager manager) {
- this.containerManager = manager;
- }
-
- /**
- * writes a given chunk.
- *
- * @param pipeline - Name and the set of machines that make this container.
- * @param keyName - Name of the Key.
- * @param info - ChunkInfo.
- * @throws StorageContainerException
- */
- @Override
- public void writeChunk(Pipeline pipeline, String keyName, ChunkInfo info,
- byte[] data, ContainerProtos.Stage stage)
- throws StorageContainerException {
- // we don't want container manager to go away while we are writing chunks.
- containerManager.readLock();
-
- // TODO : Take keyManager Write lock here.
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData container =
- containerManager.readContainer(containerName);
- File chunkFile = ChunkUtils.validateChunk(pipeline, container, info);
- File tmpChunkFile = getTmpChunkFile(chunkFile, info);
-
- LOG.debug("writing chunk:{} chunk stage:{} chunk file:{} tmp chunk file",
- info.getChunkName(), stage, chunkFile, tmpChunkFile);
- switch (stage) {
- case WRITE_DATA:
- ChunkUtils.writeData(tmpChunkFile, info, data);
- break;
- case COMMIT_DATA:
- commitChunk(tmpChunkFile, chunkFile, containerName, info.getLen());
- break;
- case COMBINED:
- // directly write to the chunk file
- long oldSize = chunkFile.length();
- ChunkUtils.writeData(chunkFile, info, data);
- long newSize = chunkFile.length();
- containerManager.incrBytesUsed(containerName, newSize - oldSize);
- containerManager.incrWriteCount(containerName);
- containerManager.incrWriteBytes(containerName, info.getLen());
- break;
- }
- } catch (ExecutionException | NoSuchAlgorithmException | IOException e) {
- LOG.error("write data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ", e,
- CONTAINER_INTERNAL_ERROR);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("write data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ", e,
- CONTAINER_INTERNAL_ERROR);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- // Create a temporary file in the same container directory
- // in the format "<chunkname>.tmp"
- private static File getTmpChunkFile(File chunkFile, ChunkInfo info)
- throws StorageContainerException {
- return new File(chunkFile.getParent(),
- chunkFile.getName() +
- OzoneConsts.CONTAINER_CHUNK_NAME_DELIMITER +
- OzoneConsts.CONTAINER_TEMPORARY_CHUNK_PREFIX);
- }
-
- // Commit the chunk by renaming the temporary chunk file to chunk file
- private void commitChunk(File tmpChunkFile, File chunkFile,
- String containerName, long chunkLen) throws IOException {
- long sizeDiff = tmpChunkFile.length() - chunkFile.length();
- // It is safe to replace here as the earlier chunk if existing should be
- // caught as part of validateChunk
- Files.move(tmpChunkFile.toPath(), chunkFile.toPath(),
- StandardCopyOption.REPLACE_EXISTING);
- containerManager.incrBytesUsed(containerName, sizeDiff);
- containerManager.incrWriteCount(containerName);
- containerManager.incrWriteBytes(containerName, chunkLen);
- }
-
- /**
- * reads the data defined by a chunk.
- *
- * @param pipeline - container pipeline.
- * @param keyName - Name of the Key
- * @param info - ChunkInfo.
- * @return byte array
- * @throws StorageContainerException
- * TODO: Right now we do not support partial reads and writes of chunks.
- * TODO: Explore if we need to do that for ozone.
- */
- @Override
- public byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info)
- throws StorageContainerException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData container =
- containerManager.readContainer(containerName);
- File chunkFile = ChunkUtils.getChunkFile(pipeline, container, info);
- ByteBuffer data = ChunkUtils.readData(chunkFile, info);
- containerManager.incrReadCount(containerName);
- containerManager.incrReadBytes(containerName, chunkFile.length());
- return data.array();
- } catch (ExecutionException | NoSuchAlgorithmException e) {
- LOG.error("read data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ",
- e, CONTAINER_INTERNAL_ERROR);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- LOG.error("read data failed. error: {}", e);
- throw new StorageContainerException("Internal error: ",
- e, CONTAINER_INTERNAL_ERROR);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * Deletes a given chunk.
- *
- * @param pipeline - Pipeline.
- * @param keyName - Key Name
- * @param info - Chunk Info
- * @throws StorageContainerException
- */
- @Override
- public void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info)
- throws StorageContainerException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- File chunkFile = ChunkUtils.getChunkFile(pipeline, containerManager
- .readContainer(containerName), info);
- if ((info.getOffset() == 0) && (info.getLen() == chunkFile.length())) {
- FileUtil.fullyDelete(chunkFile);
- containerManager.decrBytesUsed(containerName, chunkFile.length());
- } else {
- LOG.error("Not Supported Operation. Trying to delete a " +
- "chunk that is in shared file. chunk info : " + info.toString());
- throw new StorageContainerException("Not Supported Operation. " +
- "Trying to delete a chunk that is in shared file. chunk info : "
- + info.toString(), UNSUPPORTED_REQUEST);
- }
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * Shutdown the chunkManager.
- *
- * In the chunkManager we haven't acquired any resources, so nothing to do
- * here. This call is made with containerManager Writelock held.
- */
- @Override
- public void shutdown() {
- Preconditions.checkState(this.containerManager.hasWriteLock());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
deleted file mode 100644
index b842493..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/ContainerLocationManagerImpl.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.container.common.interfaces
- .ContainerLocationManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerLocationManagerMXBean;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.IOException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * A class that tells the ContainerManager where to place the containers.
- * Please note : There is *no* one-to-one correlation between metadata
- * Locations and data Locations.
- *
- * For example : A user could map all container files to a
- * SSD but leave data/metadata on bunch of other disks.
- */
-public class ContainerLocationManagerImpl implements ContainerLocationManager,
- ContainerLocationManagerMXBean {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerLocationManagerImpl.class);
-
- private final List<ContainerStorageLocation> dataLocations;
- private int currentIndex;
- private final List<StorageLocation> metadataLocations;
- private final ObjectName jmxbean;
-
- /**
- * Constructs a Location Manager.
- * @param metadataLocations - Refers to the metadataLocations
- * where we store the container metadata.
- * @param dataDirs - metadataLocations where we store the actual
- * data or chunk files.
- * @param conf - configuration.
- * @throws IOException
- */
- public ContainerLocationManagerImpl(List<StorageLocation> metadataLocations,
- List<StorageLocation> dataDirs, Configuration conf)
- throws IOException {
- dataLocations = new LinkedList<>();
- for (StorageLocation dataDir : dataDirs) {
- dataLocations.add(new ContainerStorageLocation(dataDir, conf));
- }
- this.metadataLocations = metadataLocations;
- jmxbean = MBeans.register("OzoneDataNode",
- ContainerLocationManager.class.getSimpleName(), this);
- }
-
- /**
- * Returns the path where the container should be placed from a set of
- * metadataLocations.
- *
- * @return A path where we should place this container and metadata.
- * @throws IOException
- */
- @Override
- public Path getContainerPath()
- throws IOException {
- Preconditions.checkState(metadataLocations.size() > 0);
- int index = currentIndex % metadataLocations.size();
- return Paths.get(metadataLocations.get(index).getNormalizedUri());
- }
-
- /**
- * Returns the path where the container Data file are stored.
- *
- * @return a path where we place the LevelDB and data files of a container.
- * @throws IOException
- */
- @Override
- public Path getDataPath(String containerName) throws IOException {
- Path currentPath = Paths.get(
- dataLocations.get(currentIndex++ % dataLocations.size())
- .getNormalizedUri());
- currentPath = currentPath.resolve(OzoneConsts.CONTAINER_PREFIX);
- return currentPath.resolve(containerName);
- }
-
- @Override
- public StorageLocationReport[] getLocationReport() throws IOException {
- StorageLocationReport[] reports =
- new StorageLocationReport[dataLocations.size()];
- for (int idx = 0; idx < dataLocations.size(); idx++) {
- ContainerStorageLocation loc = dataLocations.get(idx);
- long scmUsed = 0;
- long remaining = 0;
- try {
- scmUsed = loc.getScmUsed();
- remaining = loc.getAvailable();
- } catch (IOException ex) {
- LOG.warn("Failed to get scmUsed and remaining for container " +
- "storage location {}", loc.getNormalizedUri());
- // reset scmUsed and remaining if df/du failed.
- scmUsed = 0;
- remaining = 0;
- }
-
- // TODO: handle failed storage
- // For now, include storage report for location that failed to get df/du.
- StorageLocationReport r = new StorageLocationReport(
- loc.getStorageUuId(), false, loc.getCapacity(),
- scmUsed, remaining);
- reports[idx] = r;
- }
- return reports;
- }
-
- /**
- * Supports clean shutdown of container location du threads.
- *
- * @throws IOException
- */
- @Override
- public void shutdown() throws IOException {
- for (ContainerStorageLocation loc: dataLocations) {
- loc.shutdown();
- }
- MBeans.unregister(jmxbean);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
deleted file mode 100644
index 8acca8a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/KeySpaceManagerProtocolPB.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm.protocolPB;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.KeySpaceManagerService;
-
-/**
- * Protocol used to communicate with KSM.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.ozone.protocol.KeySpaceManagerProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface KeySpaceManagerProtocolPB
- extends KeySpaceManagerService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
deleted file mode 100644
index 67f9f7b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/ksm/protocolPB/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm.protocolPB;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
deleted file mode 100644
index 764ff3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone;
-
-/**
- This package contains ozone client side libraries.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
deleted file mode 100644
index fdc3ce7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/KSMPBHelper.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclType;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.OzoneAclInfo.OzoneAclRights;
-
-/**
- * Utilities for converting protobuf classes.
- */
-public final class KSMPBHelper {
-
- private KSMPBHelper() {
- /** Hidden constructor */
- }
-
- /**
- * Converts OzoneAcl into protobuf's OzoneAclInfo.
- * @return OzoneAclInfo
- */
- public static OzoneAclInfo convertOzoneAcl(OzoneAcl acl) {
- OzoneAclInfo.OzoneAclType aclType;
- switch(acl.getType()) {
- case USER:
- aclType = OzoneAclType.USER;
- break;
- case GROUP:
- aclType = OzoneAclType.GROUP;
- break;
- case WORLD:
- aclType = OzoneAclType.WORLD;
- break;
- default:
- throw new IllegalArgumentException("ACL type is not recognized");
- }
- OzoneAclInfo.OzoneAclRights aclRights;
- switch(acl.getRights()) {
- case READ:
- aclRights = OzoneAclRights.READ;
- break;
- case WRITE:
- aclRights = OzoneAclRights.WRITE;
- break;
- case READ_WRITE:
- aclRights = OzoneAclRights.READ_WRITE;
- break;
- default:
- throw new IllegalArgumentException("ACL right is not recognized");
- }
-
- return OzoneAclInfo.newBuilder().setType(aclType)
- .setName(acl.getName())
- .setRights(aclRights)
- .build();
- }
-
- /**
- * Converts protobuf's OzoneAclInfo into OzoneAcl.
- * @return OzoneAcl
- */
- public static OzoneAcl convertOzoneAcl(OzoneAclInfo aclInfo) {
- OzoneAcl.OzoneACLType aclType;
- switch(aclInfo.getType()) {
- case USER:
- aclType = OzoneAcl.OzoneACLType.USER;
- break;
- case GROUP:
- aclType = OzoneAcl.OzoneACLType.GROUP;
- break;
- case WORLD:
- aclType = OzoneAcl.OzoneACLType.WORLD;
- break;
- default:
- throw new IllegalArgumentException("ACL type is not recognized");
- }
- OzoneAcl.OzoneACLRights aclRights;
- switch(aclInfo.getRights()) {
- case READ:
- aclRights = OzoneAcl.OzoneACLRights.READ;
- break;
- case WRITE:
- aclRights = OzoneAcl.OzoneACLRights.WRITE;
- break;
- case READ_WRITE:
- aclRights = OzoneAcl.OzoneACLRights.READ_WRITE;
- break;
- default:
- throw new IllegalArgumentException("ACL right is not recognized");
- }
-
- return new OzoneAcl(aclType, aclInfo.getName(), aclRights);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
deleted file mode 100644
index 860386d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocolPB;
-
-/**
- * This package contains classes for the Protocol Buffers binding of Ozone
- * protocols.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
deleted file mode 100644
index a51d3b7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/ContainerID.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.math3.util.MathUtils;
-
-/**
- * Container ID is an integer that is a value between 1..MAX_CONTAINER ID.
- * <p>
- * We are creating a specific type for this to avoid mixing this with
- * normal integers in code.
- */
-public class ContainerID implements Comparable {
-
- private final long id;
-
- /**
- * Constructs ContainerID.
- *
- * @param id int
- */
- public ContainerID(long id) {
- Preconditions.checkState(id > 0,
- "Container ID should be a positive int");
- this.id = id;
- }
-
- /**
- * Factory method for creation of ContainerID.
- * @param containerID long
- * @return ContainerID.
- */
- public static ContainerID valueof(long containerID) {
- Preconditions.checkState(containerID > 0);
- return new ContainerID(containerID);
- }
-
- /**
- * Returns int representation of ID.
- *
- * @return int
- */
- public long getId() {
- return id;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- ContainerID that = (ContainerID) o;
-
- return id == that.id;
- }
-
- @Override
- public int hashCode() {
- return MathUtils.hash(id);
- }
-
- @Override
- public int compareTo(Object o) {
- Preconditions.checkNotNull(o);
- if (o instanceof ContainerID) {
- return Long.compare(((ContainerID) o).getId(), this.getId());
- }
- throw new IllegalArgumentException("Object O, should be an instance " +
- "of ContainerID");
- }
-
- @Override
- public String toString() {
- return "id=" + id;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
deleted file mode 100644
index 61f5609..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/scm/container/ContainerStates/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * Container States.
- */
-package org.apache.hadoop.ozone.scm.container.ContainerStates;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
deleted file mode 100644
index fbe9637..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmConfigKeys.java
+++ /dev/null
@@ -1,260 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-/**
- * This class contains constants for configuration keys used in SCM.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class ScmConfigKeys {
-
- public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY =
- "scm.container.client.idle.threshold";
- public static final String SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT =
- "10s";
-
- public static final String SCM_CONTAINER_CLIENT_MAX_SIZE_KEY =
- "scm.container.client.max.size";
- public static final int SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT =
- 256;
-
- public static final String SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS =
- "scm.container.client.max.outstanding.requests";
- public static final int SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS_DEFAULT
- = 100;
-
- public static final String DFS_CONTAINER_RATIS_ENABLED_KEY
- = "dfs.container.ratis.enabled";
- public static final boolean DFS_CONTAINER_RATIS_ENABLED_DEFAULT
- = false;
- public static final String DFS_CONTAINER_RATIS_RPC_TYPE_KEY
- = "dfs.container.ratis.rpc.type";
- public static final String DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT
- = "GRPC";
- public static final String DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY
- = "dfs.container.ratis.num.write.chunk.threads";
- public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT
- = 60;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY =
- "dfs.container.ratis.segment.size";
- public static final int DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT =
- 1 * 1024 * 1024 * 1024;
- public static final String DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY =
- "dfs.container.ratis.segment.preallocated.size";
- public static final int
- DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT = 128 * 1024 * 1024;
-
- // TODO : this is copied from OzoneConsts, may need to move to a better place
- public static final String OZONE_SCM_CHUNK_SIZE_KEY = "ozone.scm.chunk.size";
- // 16 MB by default
- public static final int OZONE_SCM_CHUNK_SIZE_DEFAULT = 16 * 1024 * 1024;
- public static final int OZONE_SCM_CHUNK_MAX_SIZE = 32 * 1024 * 1024;
-
- public static final String OZONE_SCM_CLIENT_PORT_KEY =
- "ozone.scm.client.port";
- public static final int OZONE_SCM_CLIENT_PORT_DEFAULT = 9860;
-
- public static final String OZONE_SCM_DATANODE_PORT_KEY =
- "ozone.scm.datanode.port";
- public static final int OZONE_SCM_DATANODE_PORT_DEFAULT = 9861;
-
- // OZONE_KSM_PORT_DEFAULT = 9862
- public static final String OZONE_SCM_BLOCK_CLIENT_PORT_KEY =
- "ozone.scm.block.client.port";
- public static final int OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT = 9863;
-
- // Container service client
- public static final String OZONE_SCM_CLIENT_ADDRESS_KEY =
- "ozone.scm.client.address";
- public static final String OZONE_SCM_CLIENT_BIND_HOST_KEY =
- "ozone.scm.client.bind.host";
- public static final String OZONE_SCM_CLIENT_BIND_HOST_DEFAULT =
- "0.0.0.0";
-
- // Block service client
- public static final String OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY =
- "ozone.scm.block.client.address";
- public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY =
- "ozone.scm.block.client.bind.host";
- public static final String OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT =
- "0.0.0.0";
-
- public static final String OZONE_SCM_DATANODE_ADDRESS_KEY =
- "ozone.scm.datanode.address";
- public static final String OZONE_SCM_DATANODE_BIND_HOST_KEY =
- "ozone.scm.datanode.bind.host";
- public static final String OZONE_SCM_DATANODE_BIND_HOST_DEFAULT =
- "0.0.0.0";
-
- public static final String OZONE_SCM_HTTP_ENABLED_KEY =
- "ozone.scm.http.enabled";
- public static final String OZONE_SCM_HTTP_BIND_HOST_KEY =
- "ozone.scm.http-bind-host";
- public static final String OZONE_SCM_HTTPS_BIND_HOST_KEY =
- "ozone.scm.https-bind-host";
- public static final String OZONE_SCM_HTTP_ADDRESS_KEY =
- "ozone.scm.http-address";
- public static final String OZONE_SCM_HTTPS_ADDRESS_KEY =
- "ozone.scm.https-address";
- public static final String OZONE_SCM_KEYTAB_FILE =
- "ozone.scm.keytab.file";
- public static final String OZONE_SCM_HTTP_BIND_HOST_DEFAULT = "0.0.0.0";
- public static final int OZONE_SCM_HTTP_BIND_PORT_DEFAULT = 9876;
- public static final int OZONE_SCM_HTTPS_BIND_PORT_DEFAULT = 9877;
-
-
- public static final String OZONE_SCM_HANDLER_COUNT_KEY =
- "ozone.scm.handler.count.key";
- public static final int OZONE_SCM_HANDLER_COUNT_DEFAULT = 10;
-
- public static final String OZONE_SCM_HEARTBEAT_INTERVAL =
- "ozone.scm.heartbeat.interval";
- public static final String OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT =
- "30s";
-
- public static final String OZONE_SCM_DEADNODE_INTERVAL =
- "ozone.scm.dead.node.interval";
- public static final String OZONE_SCM_DEADNODE_INTERVAL_DEFAULT =
- "10m";
-
- public static final String OZONE_SCM_MAX_HB_COUNT_TO_PROCESS =
- "ozone.scm.max.hb.count.to.process";
- public static final int OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT = 5000;
-
- public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL =
- "ozone.scm.heartbeat.thread.interval";
- public static final String OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT =
- "3s";
-
- public static final String OZONE_SCM_STALENODE_INTERVAL =
- "ozone.scm.stale.node.interval";
- public static final String OZONE_SCM_STALENODE_INTERVAL_DEFAULT =
- "90s";
-
- public static final String OZONE_SCM_HEARTBEAT_RPC_TIMEOUT =
- "ozone.scm.heartbeat.rpc-timeout";
- public static final long OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT =
- 1000;
-
- /**
- * Defines how frequently we will log the missing of heartbeat to a specific
- * SCM. In the default case we will write a warning message for each 10
- * sequential heart beats that we miss to a specific SCM. This is to avoid
- * overrunning the log with lots of HB missed Log statements.
- */
- public static final String OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT =
- "ozone.scm.heartbeat.log.warn.interval.count";
- public static final int OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT =
- 10;
-
- // ozone.scm.names key is a set of DNS | DNS:PORT | IP Address | IP:PORT.
- // Written as a comma separated string. e.g. scm1, scm2:8020, 7.7.7.7:7777
- //
- // If this key is not specified datanodes will not be able to find
- // SCM. The SCM membership can be dynamic, so this key should contain
- // all possible SCM names. Once the SCM leader is discovered datanodes will
- // get the right list of SCMs to heartbeat to from the leader.
- // While it is good for the datanodes to know the names of all SCM nodes,
- // it is sufficient to actually know the name of on working SCM. That SCM
- // will be able to return the information about other SCMs that are part of
- // the SCM replicated Log.
- //
- //In case of a membership change, any one of the SCM machines will be
- // able to send back a new list to the datanodes.
- public static final String OZONE_SCM_NAMES = "ozone.scm.names";
-
- public static final int OZONE_SCM_DEFAULT_PORT =
- OZONE_SCM_DATANODE_PORT_DEFAULT;
- // File Name and path where datanode ID is to written to.
- // if this value is not set then container startup will fail.
- public static final String OZONE_SCM_DATANODE_ID = "ozone.scm.datanode.id";
-
- public static final String OZONE_SCM_DATANODE_ID_PATH_DEFAULT = "datanode.id";
-
- public static final String OZONE_SCM_DB_CACHE_SIZE_MB =
- "ozone.scm.db.cache.size.mb";
- public static final int OZONE_SCM_DB_CACHE_SIZE_DEFAULT = 128;
-
- public static final String OZONE_SCM_CONTAINER_SIZE_GB =
- "ozone.scm.container.size.gb";
- public static final int OZONE_SCM_CONTAINER_SIZE_DEFAULT = 5;
-
- public static final String OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY =
- "ozone.scm.container.placement.impl";
-
- public static final String OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE =
- "ozone.scm.container.provision_batch_size";
- public static final int OZONE_SCM_CONTAINER_PROVISION_BATCH_SIZE_DEFAULT = 20;
-
- public static final String OZONE_SCM_CONTAINER_DELETION_CHOOSING_POLICY =
- "ozone.scm.container.deletion-choosing.policy";
-
- public static final String OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT =
- "ozone.scm.container.creation.lease.timeout";
-
- public static final String
- OZONE_SCM_CONTAINER_CREATION_LEASE_TIMEOUT_DEFAULT = "60s";
-
- /**
- * Don't start processing a pool if we have not had a minimum number of
- * seconds from the last processing.
- */
- public static final String OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL =
- "ozone.scm.container.report.processing.interval";
- public static final String
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT = "60s";
-
- /**
- * This determines the total number of pools to be processed in parallel.
- */
- public static final String OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS =
- "ozone.scm.max.nodepool.processing.threads";
- public static final int OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT = 1;
- /**
- * These 2 settings control the number of threads in executor pool and time
- * outs for thw container reports from all nodes.
- */
- public static final String OZONE_SCM_MAX_CONTAINER_REPORT_THREADS =
- "ozone.scm.max.container.report.threads";
- public static final int OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT = 100;
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT =
- "ozone.scm.container.reports.wait.timeout";
- public static final String OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT =
- "5m";
-
- public static final String OZONE_SCM_BLOCK_DELETION_MAX_RETRY =
- "ozone.scm.block.deletion.max.retry";
- public static final int OZONE_SCM_BLOCK_DELETION_MAX_RETRY_DEFAULT = 4096;
-
- // Once a container usage crosses this threshold, it is eligible for
- // closing.
- public static final String OZONE_SCM_CONTAINER_CLOSE_THRESHOLD =
- "ozone.scm.container.close.threshold";
- public static final float OZONE_SCM_CONTAINER_CLOSE_THRESHOLD_DEFAULT = 0.9f;
-
- /**
- * Never constructed.
- */
- private ScmConfigKeys() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
deleted file mode 100644
index e442fe2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/ScmInfo.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-/**
- * ScmInfo wraps the result returned from SCM#getScmInfo which
- * contains clusterId and the SCM Id.
- */
-public final class ScmInfo {
- private String clusterId;
- private String scmId;
-
- /**
- * Builder for ScmInfo.
- */
- public static class Builder {
- private String clusterId;
- private String scmId;
-
- /**
- * sets the cluster id.
- * @param cid clusterId to be set
- * @return Builder for ScmInfo
- */
- public Builder setClusterId(String cid) {
- this.clusterId = cid;
- return this;
- }
-
- /**
- * sets the scmId.
- * @param id scmId
- * @return Builder for scmInfo
- */
- public Builder setScmId(String id) {
- this.scmId = id;
- return this;
- }
-
- public ScmInfo build() {
- return new ScmInfo(clusterId, scmId);
- }
- }
-
- private ScmInfo(String clusterId, String scmId) {
- this.clusterId = clusterId;
- this.scmId = scmId;
- }
-
- /**
- * Gets the clusterId from the Version file.
- * @return ClusterId
- */
- public String getClusterId() {
- return clusterId;
- }
-
- /**
- * Gets the SCM Id from the Version file.
- * @return SCM Id
- */
- public String getScmId() {
- return scmId;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
deleted file mode 100644
index bde9064..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClient.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import io.netty.bootstrap.Bootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.List;
-import java.util.concurrent.Semaphore;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public class XceiverClient extends XceiverClientSpi {
- static final Logger LOG = LoggerFactory.getLogger(XceiverClient.class);
- private final Pipeline pipeline;
- private final Configuration config;
- private Channel channel;
- private Bootstrap b;
- private EventLoopGroup group;
- private final Semaphore semaphore;
-
- /**
- * Constructs a client that can communicate with the Container framework on
- * data nodes.
- *
- * @param pipeline - Pipeline that defines the machines.
- * @param config -- Ozone Config
- */
- public XceiverClient(Pipeline pipeline, Configuration config) {
- super();
- Preconditions.checkNotNull(pipeline);
- Preconditions.checkNotNull(config);
- this.pipeline = pipeline;
- this.config = config;
- this.semaphore =
- new Semaphore(OzoneClientUtils.getMaxOutstandingRequests(config));
- }
-
- @Override
- public void connect() throws Exception {
- if (channel != null && channel.isActive()) {
- throw new IOException("This client is already connected to a host.");
- }
-
- group = new NioEventLoopGroup();
- b = new Bootstrap();
- b.group(group)
- .channel(NioSocketChannel.class)
- .handler(new LoggingHandler(LogLevel.INFO))
- .handler(new XceiverClientInitializer(this.pipeline, semaphore));
- DatanodeID leader = this.pipeline.getLeader();
-
- // read port from the data node, on failure use default configured
- // port.
- int port = leader.getContainerPort();
- if (port == 0) {
- port = config.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
- }
- LOG.debug("Connecting to server Port : " + port);
- channel = b.connect(leader.getHostName(), port).sync().channel();
- }
-
- /**
- * Returns if the exceiver client connects to a server.
- *
- * @return True if the connection is alive, false otherwise.
- */
- @VisibleForTesting
- public boolean isConnected() {
- return channel.isActive();
- }
-
- @Override
- public void close() {
- if (group != null) {
- group.shutdownGracefully().awaitUninterruptibly();
- }
- }
-
- @Override
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- @Override
- public ContainerProtos.ContainerCommandResponseProto sendCommand(
- ContainerProtos.ContainerCommandRequestProto request) throws IOException {
- try {
- if ((channel == null) || (!channel.isActive())) {
- throw new IOException("This channel is not connected.");
- }
- XceiverClientHandler handler =
- channel.pipeline().get(XceiverClientHandler.class);
-
- return handler.sendCommand(request);
- } catch (ExecutionException | InterruptedException e) {
- /**
- * In case the netty channel handler throws an exception,
- * the exception thrown will be wrapped within {@link ExecutionException}.
- * Unwarpping here so that original exception gets passed
- * to to the client.
- */
- if (e instanceof ExecutionException) {
- Throwable cause = e.getCause();
- if (cause instanceof IOException) {
- throw (IOException) cause;
- }
- }
- throw new IOException(
- "Unexpected exception during execution:" + e.getMessage());
- }
- }
-
- /**
- * Sends a given command to server gets a waitable future back.
- *
- * @param request Request
- * @return Response to the command
- * @throws IOException
- */
- @Override
- public CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
- sendCommandAsync(ContainerProtos.ContainerCommandRequestProto request)
- throws IOException, ExecutionException, InterruptedException {
- if ((channel == null) || (!channel.isActive())) {
- throw new IOException("This channel is not connected.");
- }
- XceiverClientHandler handler =
- channel.pipeline().get(XceiverClientHandler.class);
- return handler.sendCommandAsync(request);
- }
-
- /**
- * Create a pipeline.
- *
- * @param pipelineID - Name of the pipeline.
- * @param datanodes - Datanodes
- */
- @Override
- public void createPipeline(String pipelineID, List<DatanodeID> datanodes)
- throws IOException {
- // For stand alone pipeline, there is no notion called setup pipeline.
- return;
- }
-
- /**
- * Returns pipeline Type.
- *
- * @return - Stand Alone as the type.
- */
- @Override
- public OzoneProtos.ReplicationType getPipelineType() {
- return OzoneProtos.ReplicationType.STAND_ALONE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
deleted file mode 100644
index ac2cf1a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientHandler.java
+++ /dev/null
@@ -1,204 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import com.google.common.base.Preconditions;
-
-import io.netty.channel.Channel;
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandResponseProto;
-
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Iterator;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
-import java.util.concurrent.Semaphore;
-
-/**
- * Netty client handler.
- */
-public class XceiverClientHandler extends
- SimpleChannelInboundHandler<ContainerCommandResponseProto> {
-
- static final Logger LOG = LoggerFactory.getLogger(XceiverClientHandler.class);
- private final ConcurrentMap<String, ResponseFuture> responses =
- new ConcurrentHashMap<>();
-
- private final Pipeline pipeline;
- private volatile Channel channel;
- private XceiverClientMetrics metrics;
- private final Semaphore semaphore;
-
- /**
- * Constructs a client that can communicate to a container server.
- */
- public XceiverClientHandler(Pipeline pipeline, Semaphore semaphore) {
- super(false);
- Preconditions.checkNotNull(pipeline);
- this.pipeline = pipeline;
- this.metrics = XceiverClientManager.getXceiverClientMetrics();
- this.semaphore = semaphore;
- }
-
- /**
- * <strong>Please keep in mind that this method will be renamed to {@code
- * messageReceived(ChannelHandlerContext, I)} in 5.0.</strong>
- * <p>
- * Is called for each message of type {@link ContainerProtos
- * .ContainerCommandResponseProto}.
- *
- * @param ctx the {@link ChannelHandlerContext} which this {@link
- * SimpleChannelInboundHandler} belongs to
- * @param msg the message to handle
- * @throws Exception is thrown if an error occurred
- */
- @Override
- public void channelRead0(ChannelHandlerContext ctx,
- ContainerProtos.ContainerCommandResponseProto msg)
- throws Exception {
- Preconditions.checkNotNull(msg);
- metrics.decrPendingContainerOpsMetrics(msg.getCmdType());
-
- String key = msg.getTraceID();
- ResponseFuture response = responses.remove(key);
- semaphore.release();
-
- if (response != null) {
- response.getFuture().complete(msg);
-
- long requestTime = response.getRequestTime();
- metrics.addContainerOpsLatency(msg.getCmdType(),
- Time.monotonicNowNanos() - requestTime);
- } else {
- LOG.error("A reply received for message that was not queued. trace " +
- "ID: {}", msg.getTraceID());
- }
- }
-
- @Override
- public void channelRegistered(ChannelHandlerContext ctx) {
- LOG.debug("channelRegistered: Connected to ctx");
- channel = ctx.channel();
- }
-
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause) {
- LOG.info("Exception in client " + cause.toString());
- Iterator<String> keyIterator = responses.keySet().iterator();
- while (keyIterator.hasNext()) {
- ResponseFuture response = responses.remove(keyIterator.next());
- response.getFuture().completeExceptionally(cause);
- semaphore.release();
- }
- ctx.close();
- }
-
- /**
- * Since netty is async, we send a work request and then wait until a response
- * appears in the reply queue. This is simple sync interface for clients. we
- * should consider building async interfaces for client if this turns out to
- * be a performance bottleneck.
- *
- * @param request - request.
- * @return -- response
- */
-
- public ContainerCommandResponseProto sendCommand(
- ContainerProtos.ContainerCommandRequestProto request)
- throws ExecutionException, InterruptedException {
- Future<ContainerCommandResponseProto> future = sendCommandAsync(request);
- return future.get();
- }
-
- /**
- * SendCommandAsyc queues a command to the Netty Subsystem and returns a
- * CompletableFuture. This Future is marked compeleted in the channelRead0
- * when the call comes back.
- * @param request - Request to execute
- * @return CompletableFuture
- */
- public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
- ContainerProtos.ContainerCommandRequestProto request)
- throws InterruptedException {
-
- // Throw an exception of request doesn't have traceId
- if (StringUtils.isEmpty(request.getTraceID())) {
- throw new IllegalArgumentException("Invalid trace ID");
- }
-
- // Setting the datanode ID in the commands, so that we can distinguish
- // commands when the cluster simulator is running.
- if(!request.hasDatanodeID()) {
- throw new IllegalArgumentException("Invalid Datanode ID");
- }
-
- metrics.incrPendingContainerOpsMetrics(request.getCmdType());
-
- CompletableFuture<ContainerCommandResponseProto> future
- = new CompletableFuture<>();
- ResponseFuture response = new ResponseFuture(future,
- Time.monotonicNowNanos());
- semaphore.acquire();
- ResponseFuture previous = responses.putIfAbsent(
- request.getTraceID(), response);
- if (previous != null) {
- LOG.error("Command with Trace already exists. Ignoring this command. " +
- "{}. Previous Command: {}", request.getTraceID(),
- previous.toString());
- throw new IllegalStateException("Duplicate trace ID. Command with this " +
- "trace ID is already executing. Please ensure that " +
- "trace IDs are not reused. ID: " + request.getTraceID());
- }
-
- channel.writeAndFlush(request);
- return response.getFuture();
- }
-
- /**
- * Class wraps response future info.
- */
- static class ResponseFuture {
- private final long requestTime;
- private final CompletableFuture<ContainerCommandResponseProto> future;
-
- ResponseFuture(CompletableFuture<ContainerCommandResponseProto> future,
- long requestTime) {
- this.future = future;
- this.requestTime = requestTime;
- }
-
- public long getRequestTime() {
- return requestTime;
- }
-
- public CompletableFuture<ContainerCommandResponseProto> getFuture() {
- return future;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
deleted file mode 100644
index 6aac960..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientInitializer.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.protobuf.ProtobufDecoder;
-import io.netty.handler.codec.protobuf.ProtobufEncoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.util.concurrent.Semaphore;
-
-/**
- * Setup the netty pipeline.
- */
-public class XceiverClientInitializer extends
- ChannelInitializer<SocketChannel> {
- private final Pipeline pipeline;
- private final Semaphore semaphore;
-
- /**
- * Constructs an Initializer for the client pipeline.
- * @param pipeline - Pipeline.
- */
- public XceiverClientInitializer(Pipeline pipeline, Semaphore semaphore) {
- this.pipeline = pipeline;
- this.semaphore = semaphore;
- }
-
- /**
- * This method will be called once when the Channel is registered. After
- * the method returns this instance will be removed from the
- * ChannelPipeline of the Channel.
- *
- * @param ch Channel which was registered.
- * @throws Exception is thrown if an error occurs. In that case the
- * Channel will be closed.
- */
- @Override
- protected void initChannel(SocketChannel ch) throws Exception {
- ChannelPipeline p = ch.pipeline();
-
- p.addLast(new ProtobufVarint32FrameDecoder());
- p.addLast(new ProtobufDecoder(ContainerProtos
- .ContainerCommandResponseProto.getDefaultInstance()));
-
- p.addLast(new ProtobufVarint32LengthFieldPrepender());
- p.addLast(new ProtobufEncoder());
-
- p.addLast(new XceiverClientHandler(this.pipeline, this.semaphore));
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
deleted file mode 100644
index 161cdce..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientManager.java
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.Callable;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-import com.google.common.cache.Cache;
-import com.google.common.cache.CacheBuilder;
-import com.google.common.cache.RemovalListener;
-import com.google.common.cache.RemovalNotification;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
- .SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .SCM_CONTAINER_CLIENT_MAX_SIZE_KEY;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.protocol.proto.OzoneProtos
- .ReplicationType.RATIS;
-
-/**
- * XceiverClientManager is responsible for the lifecycle of XceiverClient
- * instances. Callers use this class to acquire an XceiverClient instance
- * connected to the desired container pipeline. When done, the caller also uses
- * this class to release the previously acquired XceiverClient instance.
- *
- *
- * This class caches connection to container for reuse purpose, such that
- * accessing same container frequently will be through the same connection
- * without reestablishing connection. But the connection will be closed if
- * not being used for a period of time.
- */
-public class XceiverClientManager implements Closeable {
-
- //TODO : change this to SCM configuration class
- private final Configuration conf;
- private final Cache<String, XceiverClientSpi> clientCache;
- private final boolean useRatis;
-
- private static XceiverClientMetrics metrics;
- /**
- * Creates a new XceiverClientManager.
- *
- * @param conf configuration
- */
- public XceiverClientManager(Configuration conf) {
- Preconditions.checkNotNull(conf);
- int maxSize = conf.getInt(SCM_CONTAINER_CLIENT_MAX_SIZE_KEY,
- SCM_CONTAINER_CLIENT_MAX_SIZE_DEFAULT);
- long staleThresholdMs = conf.getTimeDuration(
- SCM_CONTAINER_CLIENT_STALE_THRESHOLD_KEY,
- SCM_CONTAINER_CLIENT_STALE_THRESHOLD_DEFAULT, TimeUnit.MILLISECONDS);
- this.useRatis = conf.getBoolean(
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_KEY,
- ScmConfigKeys.DFS_CONTAINER_RATIS_ENABLED_DEFAULT);
- this.conf = conf;
- this.clientCache = CacheBuilder.newBuilder()
- .expireAfterAccess(staleThresholdMs, TimeUnit.MILLISECONDS)
- .maximumSize(maxSize)
- .removalListener(
- new RemovalListener<String, XceiverClientSpi>() {
- @Override
- public void onRemoval(
- RemovalNotification<String, XceiverClientSpi>
- removalNotification) {
- synchronized (clientCache) {
- // Mark the entry as evicted
- XceiverClientSpi info = removalNotification.getValue();
- info.setEvicted();
- }
- }
- }).build();
- }
-
- @VisibleForTesting
- public Cache<String, XceiverClientSpi> getClientCache() {
- return clientCache;
- }
-
- /**
- * Acquires a XceiverClientSpi connected to a container capable of
- * storing the specified key.
- *
- * If there is already a cached XceiverClientSpi, simply return
- * the cached otherwise create a new one.
- *
- * @param pipeline the container pipeline for the client connection
- * @return XceiverClientSpi connected to a container
- * @throws IOException if a XceiverClientSpi cannot be acquired
- */
- public XceiverClientSpi acquireClient(Pipeline pipeline)
- throws IOException {
- Preconditions.checkNotNull(pipeline);
- Preconditions.checkArgument(pipeline.getMachines() != null);
- Preconditions.checkArgument(!pipeline.getMachines().isEmpty());
-
- synchronized (clientCache) {
- XceiverClientSpi info = getClient(pipeline);
- info.incrementReference();
- return info;
- }
- }
-
- /**
- * Releases a XceiverClientSpi after use.
- *
- * @param client client to release
- */
- public void releaseClient(XceiverClientSpi client) {
- Preconditions.checkNotNull(client);
- synchronized (clientCache) {
- client.decrementReference();
- }
- }
-
- private XceiverClientSpi getClient(Pipeline pipeline)
- throws IOException {
- String containerName = pipeline.getContainerName();
- try {
- return clientCache.get(containerName,
- new Callable<XceiverClientSpi>() {
- @Override
- public XceiverClientSpi call() throws Exception {
- XceiverClientSpi client = pipeline.getType() == RATIS ?
- XceiverClientRatis.newXceiverClientRatis(pipeline, conf)
- : new XceiverClient(pipeline, conf);
- client.connect();
- return client;
- }
- });
- } catch (Exception e) {
- throw new IOException(
- "Exception getting XceiverClient: " + e.toString(), e);
- }
- }
-
- /**
- * Close and remove all the cached clients.
- */
- public void close() {
- //closing is done through RemovalListener
- clientCache.invalidateAll();
- clientCache.cleanUp();
-
- if (metrics != null) {
- metrics.unRegister();
- }
- }
-
- /**
- * Tells us if Ratis is enabled for this cluster.
- * @return True if Ratis is enabled.
- */
- public boolean isUseRatis() {
- return useRatis;
- }
-
- /**
- * Returns hard coded 3 as replication factor.
- * @return 3
- */
- public OzoneProtos.ReplicationFactor getFactor() {
- if(isUseRatis()) {
- return OzoneProtos.ReplicationFactor.THREE;
- }
- return OzoneProtos.ReplicationFactor.ONE;
- }
-
- /**
- * Returns the default replication type.
- * @return Ratis or Standalone
- */
- public OzoneProtos.ReplicationType getType() {
- // TODO : Fix me and make Ratis default before release.
- // TODO: Remove this as replication factor and type are pipeline properties
- if(isUseRatis()) {
- return OzoneProtos.ReplicationType.RATIS;
- }
- return OzoneProtos.ReplicationType.STAND_ALONE;
- }
-
- /**
- * Get xceiver client metric.
- */
- public synchronized static XceiverClientMetrics getXceiverClientMetrics() {
- if (metrics == null) {
- metrics = XceiverClientMetrics.create();
- }
-
- return metrics;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
deleted file mode 100644
index 6359db1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientMetrics.java
+++ /dev/null
@@ -1,92 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.scm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MetricsRegistry;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * The client metrics for the Storage Container protocol.
- */
-@InterfaceAudience.Private
-@Metrics(about = "Storage Container Client Metrics", context = "dfs")
-public class XceiverClientMetrics {
- public static final String SOURCE_NAME = XceiverClientMetrics.class
- .getSimpleName();
-
- private @Metric MutableCounterLong pendingOps;
- private MutableCounterLong[] pendingOpsArray;
- private MutableRate[] containerOpsLatency;
- private MetricsRegistry registry;
-
- public XceiverClientMetrics() {
- int numEnumEntries = ContainerProtos.Type.values().length;
- this.registry = new MetricsRegistry(SOURCE_NAME);
-
- this.pendingOpsArray = new MutableCounterLong[numEnumEntries];
- this.containerOpsLatency = new MutableRate[numEnumEntries];
- for (int i = 0; i < numEnumEntries; i++) {
- pendingOpsArray[i] = registry.newCounter(
- "numPending" + ContainerProtos.Type.valueOf(i + 1),
- "number of pending" + ContainerProtos.Type.valueOf(i + 1) + " ops",
- (long) 0);
-
- containerOpsLatency[i] = registry.newRate(
- ContainerProtos.Type.valueOf(i + 1) + "Latency",
- "latency of " + ContainerProtos.Type.valueOf(i + 1)
- + " ops");
- }
- }
-
- public static XceiverClientMetrics create() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(SOURCE_NAME, "Storage Container Client Metrics",
- new XceiverClientMetrics());
- }
-
- public void incrPendingContainerOpsMetrics(ContainerProtos.Type type) {
- pendingOps.incr();
- pendingOpsArray[type.ordinal()].incr();
- }
-
- public void decrPendingContainerOpsMetrics(ContainerProtos.Type type) {
- pendingOps.incr(-1);
- pendingOpsArray[type.ordinal()].incr(-1);
- }
-
- public void addContainerOpsLatency(ContainerProtos.Type type,
- long latencyNanos) {
- containerOpsLatency[type.ordinal()].add(latencyNanos);
- }
-
- public long getContainerOpsMetrics(ContainerProtos.Type type) {
- return pendingOpsArray[type.ordinal()].value();
- }
-
- public void unRegister() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- ms.unregisterSource(SOURCE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
deleted file mode 100644
index 3bc70ed..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientRatis.java
+++ /dev/null
@@ -1,264 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.base.Preconditions;
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.client.RaftClient;
-import org.apache.ratis.protocol.RaftClientReply;
-import org.apache.ratis.protocol.RaftGroup;
-import org.apache.ratis.protocol.RaftPeer;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Objects;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicReference;
-
-/**
- * An abstract implementation of {@link XceiverClientSpi} using Ratis.
- * The underlying RPC mechanism can be chosen via the constructor.
- */
-public final class XceiverClientRatis extends XceiverClientSpi {
- static final Logger LOG = LoggerFactory.getLogger(XceiverClientRatis.class);
-
- public static XceiverClientRatis newXceiverClientRatis(
- Pipeline pipeline, Configuration ozoneConf) {
- final String rpcType = ozoneConf.get(
- ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
- ScmConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
- final int maxOutstandingRequests =
- OzoneClientUtils.getMaxOutstandingRequests(ozoneConf);
- return new XceiverClientRatis(pipeline,
- SupportedRpcType.valueOfIgnoreCase(rpcType), maxOutstandingRequests);
- }
-
- private final Pipeline pipeline;
- private final RpcType rpcType;
- private final AtomicReference<RaftClient> client = new AtomicReference<>();
- private final int maxOutstandingRequests;
-
- /**
- * Constructs a client.
- */
- private XceiverClientRatis(Pipeline pipeline, RpcType rpcType,
- int maxOutStandingChunks) {
- super();
- this.pipeline = pipeline;
- this.rpcType = rpcType;
- this.maxOutstandingRequests = maxOutStandingChunks;
- }
-
- /**
- * {@inheritDoc}
- */
- public void createPipeline(String clusterId, List<DatanodeID> datanodes)
- throws IOException {
- RaftGroup group = RatisHelper.newRaftGroup(datanodes);
- LOG.debug("initializing pipeline:{} with nodes:{}", clusterId,
- group.getPeers());
- reinitialize(datanodes, group);
- }
-
- /**
- * Returns Ratis as pipeline Type.
- *
- * @return - Ratis
- */
- @Override
- public OzoneProtos.ReplicationType getPipelineType() {
- return OzoneProtos.ReplicationType.RATIS;
- }
-
- private void reinitialize(List<DatanodeID> datanodes, RaftGroup group)
- throws IOException {
- if (datanodes.isEmpty()) {
- return;
- }
-
- IOException exception = null;
- for (DatanodeID d : datanodes) {
- try {
- reinitialize(d, group);
- } catch (IOException ioe) {
- if (exception == null) {
- exception = new IOException(
- "Failed to reinitialize some of the RaftPeer(s)", ioe);
- } else {
- exception.addSuppressed(ioe);
- }
- }
- }
- if (exception != null) {
- throw exception;
- }
- }
-
- /**
- * Adds a new peers to the Ratis Ring.
- *
- * @param datanode - new datanode
- * @param group - Raft group
- * @throws IOException - on Failure.
- */
- private void reinitialize(DatanodeID datanode, RaftGroup group)
- throws IOException {
- final RaftPeer p = RatisHelper.toRaftPeer(datanode);
- try (RaftClient client = RatisHelper.newRaftClient(rpcType, p)) {
- client.reinitialize(group, p.getId());
- } catch (IOException ioe) {
- LOG.error("Failed to reinitialize RaftPeer:{} datanode: {} ",
- p, datanode, ioe);
- throw new IOException("Failed to reinitialize RaftPeer " + p
- + "(datanode=" + datanode + ")", ioe);
- }
- }
-
- @Override
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- @Override
- public void connect() throws Exception {
- LOG.debug("Connecting to pipeline:{} leader:{}",
- getPipeline().getPipelineName(),
- RatisHelper.toRaftPeerId(pipeline.getLeader()));
- // TODO : XceiverClient ratis should pass the config value of
- // maxOutstandingRequests so as to set the upper bound on max no of async
- // requests to be handled by raft client
- if (!client.compareAndSet(null,
- RatisHelper.newRaftClient(rpcType, getPipeline()))) {
- throw new IllegalStateException("Client is already connected.");
- }
- }
-
- @Override
- public void close() {
- final RaftClient c = client.getAndSet(null);
- if (c != null) {
- try {
- c.close();
- } catch (IOException e) {
- throw new IllegalStateException(e);
- }
- }
- }
-
- private RaftClient getClient() {
- return Objects.requireNonNull(client.get(), "client is null");
- }
-
- private boolean isReadOnly(ContainerCommandRequestProto proto) {
- switch (proto.getCmdType()) {
- case ReadContainer:
- case ReadChunk:
- case ListKey:
- case GetKey:
- case GetSmallFile:
- case ListContainer:
- case ListChunk:
- return true;
- case CloseContainer:
- case WriteChunk:
- case UpdateContainer:
- case CompactChunk:
- case CreateContainer:
- case DeleteChunk:
- case DeleteContainer:
- case DeleteKey:
- case PutKey:
- case PutSmallFile:
- default:
- return false;
- }
- }
-
- private RaftClientReply sendRequest(ContainerCommandRequestProto request)
- throws IOException {
- boolean isReadOnlyRequest = isReadOnly(request);
- ByteString byteString =
- ShadedProtoUtil.asShadedByteString(request.toByteArray());
- LOG.debug("sendCommand {} {}", isReadOnlyRequest, request);
- final RaftClientReply reply = isReadOnlyRequest ?
- getClient().sendReadOnly(() -> byteString) :
- getClient().send(() -> byteString);
- LOG.debug("reply {} {}", isReadOnlyRequest, reply);
- return reply;
- }
-
- private CompletableFuture<RaftClientReply> sendRequestAsync(
- ContainerCommandRequestProto request) throws IOException {
- boolean isReadOnlyRequest = isReadOnly(request);
- ByteString byteString =
- ShadedProtoUtil.asShadedByteString(request.toByteArray());
- LOG.debug("sendCommandAsync {} {}", isReadOnlyRequest, request);
- return isReadOnlyRequest ? getClient().sendReadOnlyAsync(() -> byteString) :
- getClient().sendAsync(() -> byteString);
- }
-
- @Override
- public ContainerCommandResponseProto sendCommand(
- ContainerCommandRequestProto request) throws IOException {
- final RaftClientReply reply = sendRequest(request);
- Preconditions.checkState(reply.isSuccess());
- return ContainerCommandResponseProto.parseFrom(
- ShadedProtoUtil.asByteString(reply.getMessage().getContent()));
- }
-
- /**
- * Sends a given command to server gets a waitable future back.
- *
- * @param request Request
- * @return Response to the command
- * @throws IOException
- */
- @Override
- public CompletableFuture<ContainerCommandResponseProto> sendCommandAsync(
- ContainerCommandRequestProto request)
- throws IOException, ExecutionException, InterruptedException {
- return sendRequestAsync(request).whenComplete((reply, e) ->
- LOG.debug("received reply {} for request: {} exception: {}", request,
- reply, e))
- .thenApply(reply -> {
- try {
- return ContainerCommandResponseProto.parseFrom(
- ShadedProtoUtil.asByteString(reply.getMessage().getContent()));
- } catch (InvalidProtocolBufferException e) {
- throw new CompletionException(e);
- }
- });
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
deleted file mode 100644
index 3711a51..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/scm/XceiverClientSpi.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.scm;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * A Client for the storageContainer protocol.
- */
-public abstract class XceiverClientSpi implements Closeable {
-
- final private AtomicInteger referenceCount;
- private boolean isEvicted;
-
- XceiverClientSpi() {
- this.referenceCount = new AtomicInteger(0);
- this.isEvicted = false;
- }
-
- void incrementReference() {
- this.referenceCount.incrementAndGet();
- }
-
- void decrementReference() {
- this.referenceCount.decrementAndGet();
- cleanup();
- }
-
- void setEvicted() {
- isEvicted = true;
- cleanup();
- }
-
- // close the xceiverClient only if,
- // 1) there is no refcount on the client
- // 2) it has been evicted from the cache.
- private void cleanup() {
- if (referenceCount.get() == 0 && isEvicted) {
- close();
- }
- }
-
- @VisibleForTesting
- public int getRefcount() {
- return referenceCount.get();
- }
-
- /**
- * Connects to the leader in the pipeline.
- */
- public abstract void connect() throws Exception;
-
- @Override
- public abstract void close();
-
- /**
- * Returns the pipeline of machines that host the container used by this
- * client.
- *
- * @return pipeline of machines that host the container
- */
- public abstract Pipeline getPipeline();
-
- /**
- * Sends a given command to server and gets the reply back.
- * @param request Request
- * @return Response to the command
- * @throws IOException
- */
- public abstract ContainerCommandResponseProto sendCommand(
- ContainerCommandRequestProto request) throws IOException;
-
- /**
- * Sends a given command to server gets a waitable future back.
- *
- * @param request Request
- * @return Response to the command
- * @throws IOException
- */
- public abstract CompletableFuture<ContainerCommandResponseProto>
- sendCommandAsync(ContainerCommandRequestProto request)
- throws IOException, ExecutionException, InterruptedException;
-
- /**
- * Create a pipeline.
- *
- * @param pipelineID - Name of the pipeline.
- * @param datanodes - Datanodes
- */
- public abstract void createPipeline(String pipelineID,
- List<DatanodeID> datanodes) throws IOException;
-
- /**
- * Returns pipeline Type.
- *
- * @return - {Stand_Alone, Ratis or Chained}
- */
- public abstract OzoneProtos.ReplicationType getPipelineType();
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
deleted file mode 100644
index 957a6d9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManagerImpl.java
+++ /dev/null
@@ -1,315 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.util.Time;
-import org.iq80.leveldb.DBException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KSM bucket manager.
- */
-public class BucketManagerImpl implements BucketManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(BucketManagerImpl.class);
-
- /**
- * KSMMetadataManager is used for accessing KSM MetadataDB and ReadWriteLock.
- */
- private final KSMMetadataManager metadataManager;
-
- /**
- * Constructs BucketManager.
- * @param metadataManager
- */
- public BucketManagerImpl(KSMMetadataManager metadataManager){
- this.metadataManager = metadataManager;
- }
-
- /**
- * MetadataDB is maintained in MetadataManager and shared between
- * BucketManager and VolumeManager. (and also by KeyManager)
- *
- * BucketManager uses MetadataDB to store bucket level information.
- *
- * Keys used in BucketManager for storing data into MetadataDB
- * for BucketInfo:
- * {volume/bucket} -> bucketInfo
- *
- * Work flow of create bucket:
- *
- * -> Check if the Volume exists in metadataDB, if not throw
- * VolumeNotFoundException.
- * -> Else check if the Bucket exists in metadataDB, if so throw
- * BucketExistException
- * -> Else update MetadataDB with VolumeInfo.
- */
-
- /**
- * Creates a bucket.
- * @param bucketInfo - KsmBucketInfo.
- */
- @Override
- public void createBucket(KsmBucketInfo bucketInfo) throws IOException {
- Preconditions.checkNotNull(bucketInfo);
- metadataManager.writeLock().lock();
- String volumeName = bucketInfo.getVolumeName();
- String bucketName = bucketInfo.getBucketName();
- try {
- byte[] volumeKey = metadataManager.getVolumeKey(volumeName);
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
-
- //Check if the volume exists
- if (metadataManager.get(volumeKey) == null) {
- LOG.debug("volume: {} not found ", volumeName);
- throw new KSMException("Volume doesn't exist",
- KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
- //Check if bucket already exists
- if (metadataManager.get(bucketKey) != null) {
- LOG.debug("bucket: {} already exists ", bucketName);
- throw new KSMException("Bucket already exist",
- KSMException.ResultCodes.FAILED_BUCKET_ALREADY_EXISTS);
- }
-
- KsmBucketInfo ksmBucketInfo = KsmBucketInfo.newBuilder()
- .setVolumeName(bucketInfo.getVolumeName())
- .setBucketName(bucketInfo.getBucketName())
- .setAcls(bucketInfo.getAcls())
- .setStorageType(bucketInfo.getStorageType())
- .setIsVersionEnabled(bucketInfo.getIsVersionEnabled())
- .setCreationTime(Time.now())
- .build();
- metadataManager.put(bucketKey, ksmBucketInfo.getProtobuf().toByteArray());
-
- LOG.debug("created bucket: {} in volume: {}", bucketName, volumeName);
- } catch (IOException | DBException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Bucket creation failed for bucket:{} in volume:{}",
- bucketName, volumeName, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Returns Bucket Information.
- *
- * @param volumeName - Name of the Volume.
- * @param bucketName - Name of the Bucket.
- */
- @Override
- public KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- metadataManager.readLock().lock();
- try {
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- byte[] value = metadataManager.get(bucketKey);
- if (value == null) {
- LOG.debug("bucket: {} not found in volume: {}.", bucketName,
- volumeName);
- throw new KSMException("Bucket not found",
- KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
- return KsmBucketInfo.getFromProtobuf(BucketInfo.parseFrom(value));
- } catch (IOException | DBException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Exception while getting bucket info for bucket: {}",
- bucketName, ex);
- }
- throw ex;
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-
- /**
- * Sets bucket property from args.
- * @param args - BucketArgs.
- * @throws IOException
- */
- @Override
- public void setBucketProperty(KsmBucketArgs args) throws IOException {
- Preconditions.checkNotNull(args);
- metadataManager.writeLock().lock();
- String volumeName = args.getVolumeName();
- String bucketName = args.getBucketName();
- try {
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- //Check if volume exists
- if(metadataManager.get(metadataManager.getVolumeKey(volumeName)) ==
- null) {
- LOG.debug("volume: {} not found ", volumeName);
- throw new KSMException("Volume doesn't exist",
- KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
- byte[] value = metadataManager.get(bucketKey);
- //Check if bucket exist
- if(value == null) {
- LOG.debug("bucket: {} not found ", bucketName);
- throw new KSMException("Bucket doesn't exist",
- KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
- KsmBucketInfo oldBucketInfo = KsmBucketInfo.getFromProtobuf(
- BucketInfo.parseFrom(value));
- KsmBucketInfo.Builder bucketInfoBuilder = KsmBucketInfo.newBuilder();
- bucketInfoBuilder.setVolumeName(oldBucketInfo.getVolumeName())
- .setBucketName(oldBucketInfo.getBucketName());
-
- //Check ACLs to update
- if(args.getAddAcls() != null || args.getRemoveAcls() != null) {
- bucketInfoBuilder.setAcls(getUpdatedAclList(oldBucketInfo.getAcls(),
- args.getRemoveAcls(), args.getAddAcls()));
- LOG.debug("Updating ACLs for bucket: {} in volume: {}",
- bucketName, volumeName);
- } else {
- bucketInfoBuilder.setAcls(oldBucketInfo.getAcls());
- }
-
- //Check StorageType to update
- StorageType storageType = args.getStorageType();
- if (storageType != null) {
- bucketInfoBuilder.setStorageType(storageType);
- LOG.debug("Updating bucket storage type for bucket: {} in volume: {}",
- bucketName, volumeName);
- } else {
- bucketInfoBuilder.setStorageType(oldBucketInfo.getStorageType());
- }
-
- //Check Versioning to update
- Boolean versioning = args.getIsVersionEnabled();
- if (versioning != null) {
- bucketInfoBuilder.setIsVersionEnabled(versioning);
- LOG.debug("Updating bucket versioning for bucket: {} in volume: {}",
- bucketName, volumeName);
- } else {
- bucketInfoBuilder
- .setIsVersionEnabled(oldBucketInfo.getIsVersionEnabled());
- }
- bucketInfoBuilder.setCreationTime(oldBucketInfo.getCreationTime());
-
- metadataManager.put(bucketKey,
- bucketInfoBuilder.build().getProtobuf().toByteArray());
- } catch (IOException | DBException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Setting bucket property failed for bucket:{} in volume:{}",
- bucketName, volumeName, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * Updates the existing ACL list with remove and add ACLs that are passed.
- * Remove is done before Add.
- *
- * @param existingAcls - old ACL list.
- * @param removeAcls - ACLs to be removed.
- * @param addAcls - ACLs to be added.
- * @return updated ACL list.
- */
- private List<OzoneAcl> getUpdatedAclList(List<OzoneAcl> existingAcls,
- List<OzoneAcl> removeAcls, List<OzoneAcl> addAcls) {
- if(removeAcls != null && !removeAcls.isEmpty()) {
- existingAcls.removeAll(removeAcls);
- }
- if(addAcls != null && !addAcls.isEmpty()) {
- addAcls.stream().filter(acl -> !existingAcls.contains(acl)).forEach(
- existingAcls::add);
- }
- return existingAcls;
- }
-
- /**
- * Deletes an existing empty bucket from volume.
- * @param volumeName - Name of the volume.
- * @param bucketName - Name of the bucket.
- * @throws IOException
- */
- public void deleteBucket(String volumeName, String bucketName)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- Preconditions.checkNotNull(bucketName);
- metadataManager.writeLock().lock();
- try {
- byte[] bucketKey = metadataManager.getBucketKey(volumeName, bucketName);
- //Check if volume exists
- if (metadataManager.get(metadataManager.getVolumeKey(volumeName))
- == null) {
- LOG.debug("volume: {} not found ", volumeName);
- throw new KSMException("Volume doesn't exist",
- KSMException.ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
- //Check if bucket exist
- if (metadataManager.get(bucketKey) == null) {
- LOG.debug("bucket: {} not found ", bucketName);
- throw new KSMException("Bucket doesn't exist",
- KSMException.ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
- //Check if bucket is empty
- if (!metadataManager.isBucketEmpty(volumeName, bucketName)) {
- LOG.debug("bucket: {} is not empty ", bucketName);
- throw new KSMException("Bucket is not empty",
- KSMException.ResultCodes.FAILED_BUCKET_NOT_EMPTY);
- }
- metadataManager.delete(bucketKey);
- } catch (IOException ex) {
- if (!(ex instanceof KSMException)) {
- LOG.error("Delete bucket failed for bucket:{} in volume:{}", bucketName,
- volumeName, ex);
- }
- throw ex;
- } finally {
- metadataManager.writeLock().unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KsmBucketInfo> listBuckets(String volumeName,
- String startBucket, String bucketPrefix, int maxNumOfBuckets)
- throws IOException {
- Preconditions.checkNotNull(volumeName);
- metadataManager.readLock().lock();
- try {
- return metadataManager.listBuckets(
- volumeName, startBucket, bucketPrefix, maxNumOfBuckets);
- } finally {
- metadataManager.readLock().unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
deleted file mode 100644
index 42331f6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMXBean.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.jmx.ServiceRuntimeInfo;
-
-/**
- * This is the JMX management interface for ksm information.
- */
-@InterfaceAudience.Private
-public interface KSMMXBean extends ServiceRuntimeInfo {
-
- String getRpcPort();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
deleted file mode 100644
index f5a2d5b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManager.java
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataStore;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.locks.Lock;
-
-/**
- * KSM metadata manager interface.
- */
-public interface KSMMetadataManager {
- /**
- * Start metadata manager.
- */
- void start();
-
- /**
- * Stop metadata manager.
- */
- void stop() throws IOException;
-
- /**
- * Get metadata store.
- * @return metadata store.
- */
- @VisibleForTesting
- MetadataStore getStore();
-
- /**
- * Returns the read lock used on Metadata DB.
- * @return readLock
- */
- Lock readLock();
-
- /**
- * Returns the write lock used on Metadata DB.
- * @return writeLock
- */
- Lock writeLock();
-
- /**
- * Returns the value associated with this key.
- * @param key - key
- * @return value
- */
- byte[] get(byte[] key) throws IOException;
-
- /**
- * Puts a Key into Metadata DB.
- * @param key - key
- * @param value - value
- */
- void put(byte[] key, byte[] value) throws IOException;
-
- /**
- * Deletes a Key from Metadata DB.
- * @param key - key
- */
- void delete(byte[] key) throws IOException;
-
- /**
- * Atomic write a batch of operations.
- * @param batch
- * @throws IOException
- */
- void writeBatch(BatchOperation batch) throws IOException;
-
- /**
- * Given a volume return the corresponding DB key.
- * @param volume - Volume name
- */
- byte[] getVolumeKey(String volume);
-
- /**
- * Given a user return the corresponding DB key.
- * @param user - User name
- */
- byte[] getUserKey(String user);
-
- /**
- * Given a volume and bucket, return the corresponding DB key.
- * @param volume - User name
- * @param bucket - Bucket name
- */
- byte[] getBucketKey(String volume, String bucket);
-
- /**
- * Given a volume, bucket and a key, return the corresponding DB key.
- * @param volume - volume name
- * @param bucket - bucket name
- * @param key - key name
- * @return bytes of DB key.
- */
- byte[] getDBKeyBytes(String volume, String bucket, String key);
-
- /**
- * Returns the DB key name of a deleted key in KSM metadata store.
- * The name for a deleted key has prefix #deleting# followed by
- * the actual key name.
- * @param keyName - key name
- * @return bytes of DB key.
- */
- byte[] getDeletedKeyName(byte[] keyName);
-
- /**
- * Returns the DB key name of a open key in KSM metadata store.
- * Should be #open# prefix followed by actual key name.
- * @param keyName - key name
- * @param id - the id for this open
- * @return bytes of DB key.
- */
- byte[] getOpenKeyNameBytes(String keyName, int id);
-
- /**
- * Returns the full name of a key given volume name, bucket name and key name.
- * Generally done by padding certain delimiters.
- *
- * @param volumeName - volume name
- * @param bucketName - bucket name
- * @param keyName - key name
- * @return the full key name.
- */
- String getKeyWithDBPrefix(String volumeName, String bucketName,
- String keyName);
-
- /**
- * Given a volume, check if it is empty,
- * i.e there are no buckets inside it.
- * @param volume - Volume name
- */
- boolean isVolumeEmpty(String volume) throws IOException;
-
- /**
- * Given a volume/bucket, check if it is empty,
- * i.e there are no keys inside it.
- * @param volume - Volume name
- * @param bucket - Bucket name
- * @return true if the bucket is empty
- */
- boolean isBucketEmpty(String volume, String bucket) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link KsmBucketInfo}
- * in the given volume.
- *
- * @param volumeName
- * the name of the volume. This argument is required,
- * this method returns buckets in this given volume.
- * @param startBucket
- * the start bucket name. Only the buckets whose name is
- * after this value will be included in the result.
- * This key is excluded from the result.
- * @param bucketPrefix
- * bucket name prefix. Only the buckets whose name has
- * this prefix will be included in the result.
- * @param maxNumOfBuckets
- * the maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<KsmBucketInfo> listBuckets(String volumeName, String startBucket,
- String bucketPrefix, int maxNumOfBuckets) throws IOException;
-
- /**
- * Returns a list of keys represented by {@link KsmKeyInfo}
- * in the given bucket.
- *
- * @param volumeName
- * the name of the volume.
- * @param bucketName
- * the name of the bucket.
- * @param startKey
- * the start key name, only the keys whose name is
- * after this value will be included in the result.
- * This key is excluded from the result.
- * @param keyPrefix
- * key name prefix, only the keys whose name has
- * this prefix will be included in the result.
- * @param maxKeys
- * the maximum number of keys to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of keys.
- * @throws IOException
- */
- List<KsmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
- throws IOException;
-
- /**
- * Returns a list of volumes owned by a given user; if user is null,
- * returns all volumes.
- *
- * @param userName
- * volume owner
- * @param prefix
- * the volume prefix used to filter the listing result.
- * @param startKey
- * the start volume name determines where to start listing from,
- * this key is excluded from the result.
- * @param maxKeys
- * the maximum number of volumes to return.
- * @return a list of {@link KsmVolumeArgs}
- * @throws IOException
- */
- List<KsmVolumeArgs> listVolumes(String userName, String prefix,
- String startKey, int maxKeys) throws IOException;
-
- /**
- * Returns a list of pending deletion key info that ups to the given count.
- * Each entry is a {@link BlockGroup}, which contains the info about the
- * key name and all its associated block IDs. A pending deletion key is
- * stored with #deleting# prefix in KSM DB.
- *
- * @param count max number of keys to return.
- * @return a list of {@link BlockGroup} represent keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
- /**
- * Returns a list of all still open key info. Which contains the info about
- * the key name and all its associated block IDs. A pending open key has
- * prefix #open# in KSM DB.
- *
- * @return a list of {@link BlockGroup} representing keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getExpiredOpenKeys() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
deleted file mode 100644
index df02182..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetadataManagerImpl.java
+++ /dev/null
@@ -1,524 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Strings;
-import com.google.common.collect.Lists;
-import org.apache.commons.lang3.tuple.ImmutablePair;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfoGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException.ResultCodes;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.BucketInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.KeyInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeInfo;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos.VolumeList;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BatchOperation;
-import org.apache.hadoop.utils.MetadataKeyFilters;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Collections;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.stream.Collectors;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.DELETING_KEY_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConsts.KSM_DB_NAME;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_KEY_ID_DELIMINATOR;
-import static org.apache.hadoop.ozone.OzoneConsts.OPEN_KEY_PREFIX;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_DB_CACHE_SIZE_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_DB_CACHE_SIZE_MB;
-
-/**
- * KSM metadata manager interface.
- */
-public class KSMMetadataManagerImpl implements KSMMetadataManager {
-
- private final MetadataStore store;
- private final ReadWriteLock lock;
- private final long openKeyExpireThresholdMS;
-
- public KSMMetadataManagerImpl(OzoneConfiguration conf) throws IOException {
- File metaDir = OzoneUtils.getOzoneMetaDirPath(conf);
- final int cacheSize = conf.getInt(OZONE_KSM_DB_CACHE_SIZE_MB,
- OZONE_KSM_DB_CACHE_SIZE_DEFAULT);
- File ksmDBFile = new File(metaDir.getPath(), KSM_DB_NAME);
- this.store = MetadataStoreBuilder.newBuilder()
- .setConf(conf)
- .setDbFile(ksmDBFile)
- .setCacheSize(cacheSize * OzoneConsts.MB)
- .build();
- this.lock = new ReentrantReadWriteLock();
- this.openKeyExpireThresholdMS = 1000 * conf.getInt(
- OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS,
- OZONE_OPEN_KEY_EXPIRE_THRESHOLD_SECONDS_DEFAULT);
- }
-
- /**
- * Start metadata manager.
- */
- @Override
- public void start() {
-
- }
-
- /**
- * Stop metadata manager.
- */
- @Override
- public void stop() throws IOException {
- if (store != null) {
- store.close();
- }
- }
-
- /**
- * Get metadata store.
- * @return store - metadata store.
- */
- @VisibleForTesting
- @Override
- public MetadataStore getStore() {
- return store;
- }
-
- /**
- * Given a volume return the corresponding DB key.
- * @param volume - Volume name
- */
- public byte[] getVolumeKey(String volume) {
- String dbVolumeName = OzoneConsts.KSM_VOLUME_PREFIX + volume;
- return DFSUtil.string2Bytes(dbVolumeName);
- }
-
- /**
- * Given a user return the corresponding DB key.
- * @param user - User name
- */
- public byte[] getUserKey(String user) {
- String dbUserName = OzoneConsts.KSM_USER_PREFIX + user;
- return DFSUtil.string2Bytes(dbUserName);
- }
-
- /**
- * Given a volume and bucket, return the corresponding DB key.
- * @param volume - User name
- * @param bucket - Bucket name
- */
- public byte[] getBucketKey(String volume, String bucket) {
- String bucketKeyString = OzoneConsts.KSM_VOLUME_PREFIX + volume
- + OzoneConsts.KSM_BUCKET_PREFIX + bucket;
- return DFSUtil.string2Bytes(bucketKeyString);
- }
-
- /**
- * @param volume
- * @param bucket
- * @return
- */
- private String getBucketWithDBPrefix(String volume, String bucket) {
- StringBuffer sb = new StringBuffer();
- sb.append(OzoneConsts.KSM_VOLUME_PREFIX)
- .append(volume)
- .append(OzoneConsts.KSM_BUCKET_PREFIX);
- if (!Strings.isNullOrEmpty(bucket)) {
- sb.append(bucket);
- }
- return sb.toString();
- }
-
- @Override
- public String getKeyWithDBPrefix(String volume, String bucket, String key) {
- String keyVB = OzoneConsts.KSM_KEY_PREFIX + volume
- + OzoneConsts.KSM_KEY_PREFIX + bucket
- + OzoneConsts.KSM_KEY_PREFIX;
- return Strings.isNullOrEmpty(key) ? keyVB : keyVB + key;
- }
-
- @Override
- public byte[] getDBKeyBytes(String volume, String bucket, String key) {
- return DFSUtil.string2Bytes(getKeyWithDBPrefix(volume, bucket, key));
- }
-
- @Override
- public byte[] getDeletedKeyName(byte[] keyName) {
- return DFSUtil.string2Bytes(
- DELETING_KEY_PREFIX + DFSUtil.bytes2String(keyName));
- }
-
- @Override
- public byte[] getOpenKeyNameBytes(String keyName, int id) {
- return DFSUtil.string2Bytes(OPEN_KEY_PREFIX + id +
- OPEN_KEY_ID_DELIMINATOR + keyName);
- }
-
- /**
- * Returns the read lock used on Metadata DB.
- * @return readLock
- */
- @Override
- public Lock readLock() {
- return lock.readLock();
- }
-
- /**
- * Returns the write lock used on Metadata DB.
- * @return writeLock
- */
- @Override
- public Lock writeLock() {
- return lock.writeLock();
- }
-
- /**
- * Returns the value associated with this key.
- * @param key - key
- * @return value
- */
- @Override
- public byte[] get(byte[] key) throws IOException {
- return store.get(key);
- }
-
- /**
- * Puts a Key into Metadata DB.
- * @param key - key
- * @param value - value
- */
- @Override
- public void put(byte[] key, byte[] value) throws IOException {
- store.put(key, value);
- }
-
- /**
- * Deletes a Key from Metadata DB.
- * @param key - key
- */
- public void delete(byte[] key) throws IOException {
- store.delete(key);
- }
-
- @Override
- public void writeBatch(BatchOperation batch) throws IOException {
- this.store.writeBatch(batch);
- }
-
- /**
- * Given a volume, check if it is empty, i.e there are no buckets inside it.
- * @param volume - Volume name
- * @return true if the volume is empty
- */
- public boolean isVolumeEmpty(String volume) throws IOException {
- String dbVolumeRootName = OzoneConsts.KSM_VOLUME_PREFIX + volume
- + OzoneConsts.KSM_BUCKET_PREFIX;
- byte[] dbVolumeRootKey = DFSUtil.string2Bytes(dbVolumeRootName);
- ImmutablePair<byte[], byte[]> volumeRoot =
- store.peekAround(0, dbVolumeRootKey);
- if (volumeRoot != null) {
- return !DFSUtil.bytes2String(volumeRoot.getKey())
- .startsWith(dbVolumeRootName);
- }
- return true;
- }
-
- /**
- * Given a volume/bucket, check if it is empty,
- * i.e there are no keys inside it.
- * @param volume - Volume name
- * @param bucket - Bucket name
- * @return true if the bucket is empty
- */
- public boolean isBucketEmpty(String volume, String bucket)
- throws IOException {
- String keyRootName = getKeyWithDBPrefix(volume, bucket, null);
- byte[] keyRoot = DFSUtil.string2Bytes(keyRootName);
- ImmutablePair<byte[], byte[]> firstKey = store.peekAround(0, keyRoot);
- if (firstKey != null) {
- return !DFSUtil.bytes2String(firstKey.getKey())
- .startsWith(keyRootName);
- }
- return true;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KsmBucketInfo> listBuckets(final String volumeName,
- final String startBucket, final String bucketPrefix,
- final int maxNumOfBuckets) throws IOException {
- List<KsmBucketInfo> result = new ArrayList<>();
- if (Strings.isNullOrEmpty(volumeName)) {
- throw new KSMException("Volume name is required.",
- ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- byte[] volumeNameBytes = getVolumeKey(volumeName);
- if (store.get(volumeNameBytes) == null) {
- throw new KSMException("Volume " + volumeName + " not found.",
- ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
-
- // A bucket starts with /#volume/#bucket_prefix
- MetadataKeyFilter filter = (preKey, currentKey, nextKey) -> {
- if (currentKey != null) {
- String bucketNamePrefix =
- getBucketWithDBPrefix(volumeName, bucketPrefix);
- String bucket = DFSUtil.bytes2String(currentKey);
- return bucket.startsWith(bucketNamePrefix);
- }
- return false;
- };
-
- List<Map.Entry<byte[], byte[]>> rangeResult;
- if (!Strings.isNullOrEmpty(startBucket)) {
- // Since we are excluding start key from the result,
- // the maxNumOfBuckets is incremented.
- rangeResult = store.getSequentialRangeKVs(
- getBucketKey(volumeName, startBucket),
- maxNumOfBuckets + 1, filter);
- if (!rangeResult.isEmpty()) {
- //Remove start key from result.
- rangeResult.remove(0);
- }
- } else {
- rangeResult = store.getSequentialRangeKVs(null, maxNumOfBuckets, filter);
- }
-
- for (Map.Entry<byte[], byte[]> entry : rangeResult) {
- KsmBucketInfo info = KsmBucketInfo.getFromProtobuf(
- BucketInfo.parseFrom(entry.getValue()));
- result.add(info);
- }
- return result;
- }
-
- @Override
- public List<KsmKeyInfo> listKeys(String volumeName, String bucketName,
- String startKey, String keyPrefix, int maxKeys) throws IOException {
- List<KsmKeyInfo> result = new ArrayList<>();
- if (Strings.isNullOrEmpty(volumeName)) {
- throw new KSMException("Volume name is required.",
- ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
-
- if (Strings.isNullOrEmpty(bucketName)) {
- throw new KSMException("Bucket name is required.",
- ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
-
- byte[] bucketNameBytes = getBucketKey(volumeName, bucketName);
- if (store.get(bucketNameBytes) == null) {
- throw new KSMException("Bucket " + bucketName + " not found.",
- ResultCodes.FAILED_BUCKET_NOT_FOUND);
- }
-
- MetadataKeyFilter filter = new KeyPrefixFilter(
- getKeyWithDBPrefix(volumeName, bucketName, keyPrefix));
-
- List<Map.Entry<byte[], byte[]>> rangeResult;
- if (!Strings.isNullOrEmpty(startKey)) {
- //Since we are excluding start key from the result,
- // the maxNumOfBuckets is incremented.
- rangeResult = store.getSequentialRangeKVs(
- getDBKeyBytes(volumeName, bucketName, startKey),
- maxKeys + 1, filter);
- if (!rangeResult.isEmpty()) {
- //Remove start key from result.
- rangeResult.remove(0);
- }
- } else {
- rangeResult = store.getSequentialRangeKVs(null, maxKeys, filter);
- }
-
- for (Map.Entry<byte[], byte[]> entry : rangeResult) {
- KsmKeyInfo info = KsmKeyInfo.getFromProtobuf(
- KeyInfo.parseFrom(entry.getValue()));
- result.add(info);
- }
- return result;
- }
-
- @Override
- public List<KsmVolumeArgs> listVolumes(String userName,
- String prefix, String startKey, int maxKeys) throws IOException {
- List<KsmVolumeArgs> result = Lists.newArrayList();
- VolumeList volumes;
- if (Strings.isNullOrEmpty(userName)) {
- volumes = getAllVolumes();
- } else {
- volumes = getVolumesByUser(userName);
- }
-
- if (volumes == null || volumes.getVolumeNamesCount() == 0) {
- return result;
- }
-
- boolean startKeyFound = Strings.isNullOrEmpty(startKey);
- for (String volumeName : volumes.getVolumeNamesList()) {
- if (!Strings.isNullOrEmpty(prefix)) {
- if (!volumeName.startsWith(prefix)) {
- continue;
- }
- }
-
- if (!startKeyFound && volumeName.equals(startKey)) {
- startKeyFound = true;
- continue;
- }
- if (startKeyFound && result.size() < maxKeys) {
- byte[] volumeInfo = store.get(this.getVolumeKey(volumeName));
- if (volumeInfo == null) {
- // Could not get volume info by given volume name,
- // since the volume name is loaded from db,
- // this probably means ksm db is corrupted or some entries are
- // accidentally removed.
- throw new KSMException("Volume info not found for " + volumeName,
- ResultCodes.FAILED_VOLUME_NOT_FOUND);
- }
- VolumeInfo info = VolumeInfo.parseFrom(volumeInfo);
- KsmVolumeArgs volumeArgs = KsmVolumeArgs.getFromProtobuf(info);
- result.add(volumeArgs);
- }
- }
-
- return result;
- }
-
- private VolumeList getVolumesByUser(String userName)
- throws KSMException {
- return getVolumesByUser(getUserKey(userName));
- }
-
- private VolumeList getVolumesByUser(byte[] userNameKey)
- throws KSMException {
- VolumeList volumes = null;
- try {
- byte[] volumesInBytes = store.get(userNameKey);
- if (volumesInBytes == null) {
- // No volume found for this user, return an empty list
- return VolumeList.newBuilder().build();
- }
- volumes = VolumeList.parseFrom(volumesInBytes);
- } catch (IOException e) {
- throw new KSMException("Unable to get volumes info by the given user, "
- + "metadata might be corrupted", e,
- ResultCodes.FAILED_METADATA_ERROR);
- }
- return volumes;
- }
-
- private VolumeList getAllVolumes() throws IOException {
- // Scan all users in database
- KeyPrefixFilter filter = new KeyPrefixFilter(OzoneConsts.KSM_USER_PREFIX);
- // We are not expecting a huge number of users per cluster,
- // it should be fine to scan all users in db and return us a
- // list of volume names in string per user.
- List<Map.Entry<byte[], byte[]>> rangeKVs = store
- .getSequentialRangeKVs(null, Integer.MAX_VALUE, filter);
-
- VolumeList.Builder builder = VolumeList.newBuilder();
- for (Map.Entry<byte[], byte[]> entry : rangeKVs) {
- VolumeList volumes = this.getVolumesByUser(entry.getKey());
- builder.addAllVolumeNames(volumes.getVolumeNamesList());
- }
-
- return builder.build();
- }
-
- @Override
- public List<BlockGroup> getPendingDeletionKeys(final int count)
- throws IOException {
- List<BlockGroup> keyBlocksList = Lists.newArrayList();
- List<Map.Entry<byte[], byte[]>> rangeResult =
- store.getRangeKVs(null, count,
- MetadataKeyFilters.getDeletingKeyFilter());
- for (Map.Entry<byte[], byte[]> entry : rangeResult) {
- KsmKeyInfo info =
- KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));
- // Get block keys as a list.
- KsmKeyLocationInfoGroup latest = info.getLatestVersionLocations();
- if (latest == null) {
- return Collections.emptyList();
- }
- List<String> item = latest.getLocationList().stream()
- .map(KsmKeyLocationInfo::getBlockID)
- .collect(Collectors.toList());
- BlockGroup keyBlocks = BlockGroup.newBuilder()
- .setKeyName(DFSUtil.bytes2String(entry.getKey()))
- .addAllBlockIDs(item)
- .build();
- keyBlocksList.add(keyBlocks);
- }
- return keyBlocksList;
- }
-
- @Override
- public List<BlockGroup> getExpiredOpenKeys() throws IOException {
- List<BlockGroup> keyBlocksList = Lists.newArrayList();
- long now = Time.now();
- final MetadataKeyFilter openKeyFilter =
- new KeyPrefixFilter(OPEN_KEY_PREFIX);
- List<Map.Entry<byte[], byte[]>> rangeResult =
- store.getSequentialRangeKVs(null, Integer.MAX_VALUE,
- openKeyFilter);
- for (Map.Entry<byte[], byte[]> entry : rangeResult) {
- KsmKeyInfo info =
- KsmKeyInfo.getFromProtobuf(KeyInfo.parseFrom(entry.getValue()));
- long lastModify = info.getModificationTime();
- if (now - lastModify < this.openKeyExpireThresholdMS) {
- // consider as may still be active, not hanging.
- continue;
- }
- // Get block keys as a list.
- List<String> item = info.getLatestVersionLocations()
- .getBlocksLatestVersionOnly().stream()
- .map(KsmKeyLocationInfo::getBlockID)
- .collect(Collectors.toList());
- BlockGroup keyBlocks = BlockGroup.newBuilder()
- .setKeyName(DFSUtil.bytes2String(entry.getKey()))
- .addAllBlockIDs(item)
- .build();
- keyBlocksList.add(keyBlocks);
- }
- return keyBlocksList;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
deleted file mode 100644
index bd29012..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMMetrics.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import com.google.common.annotations.VisibleForTesting;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-
-/**
- * This class is for maintaining KeySpaceManager statistics.
- */
-@InterfaceAudience.Private
-@Metrics(about="Key Space Manager Metrics", context="dfs")
-public class KSMMetrics {
- private static final String SOURCE_NAME =
- KSMMetrics.class.getSimpleName();
-
- // KSM request type op metrics
- private @Metric MutableCounterLong numVolumeOps;
- private @Metric MutableCounterLong numBucketOps;
- private @Metric MutableCounterLong numKeyOps;
-
- // KSM op metrics
- private @Metric MutableCounterLong numVolumeCreates;
- private @Metric MutableCounterLong numVolumeUpdates;
- private @Metric MutableCounterLong numVolumeInfos;
- private @Metric MutableCounterLong numVolumeCheckAccesses;
- private @Metric MutableCounterLong numBucketCreates;
- private @Metric MutableCounterLong numVolumeDeletes;
- private @Metric MutableCounterLong numBucketInfos;
- private @Metric MutableCounterLong numBucketUpdates;
- private @Metric MutableCounterLong numBucketDeletes;
- private @Metric MutableCounterLong numKeyAllocate;
- private @Metric MutableCounterLong numKeyLookup;
- private @Metric MutableCounterLong numKeyDeletes;
- private @Metric MutableCounterLong numBucketLists;
- private @Metric MutableCounterLong numKeyLists;
- private @Metric MutableCounterLong numVolumeLists;
- private @Metric MutableCounterLong numKeyCommits;
- private @Metric MutableCounterLong numAllocateBlockCalls;
- private @Metric MutableCounterLong numGetServiceLists;
-
- // Failure Metrics
- private @Metric MutableCounterLong numVolumeCreateFails;
- private @Metric MutableCounterLong numVolumeUpdateFails;
- private @Metric MutableCounterLong numVolumeInfoFails;
- private @Metric MutableCounterLong numVolumeDeleteFails;
- private @Metric MutableCounterLong numBucketCreateFails;
- private @Metric MutableCounterLong numVolumeCheckAccessFails;
- private @Metric MutableCounterLong numBucketInfoFails;
- private @Metric MutableCounterLong numBucketUpdateFails;
- private @Metric MutableCounterLong numBucketDeleteFails;
- private @Metric MutableCounterLong numKeyAllocateFails;
- private @Metric MutableCounterLong numKeyLookupFails;
- private @Metric MutableCounterLong numKeyDeleteFails;
- private @Metric MutableCounterLong numBucketListFails;
- private @Metric MutableCounterLong numKeyListFails;
- private @Metric MutableCounterLong numVolumeListFails;
- private @Metric MutableCounterLong numKeyCommitFails;
- private @Metric MutableCounterLong numBlockAllocateCallFails;
- private @Metric MutableCounterLong numGetServiceListFails;
-
- public KSMMetrics() {
- }
-
- public static KSMMetrics create() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(SOURCE_NAME,
- "Key Space Manager Metrics",
- new KSMMetrics());
- }
-
- public void incNumVolumeCreates() {
- numVolumeOps.incr();
- numVolumeCreates.incr();
- }
-
- public void incNumVolumeUpdates() {
- numVolumeOps.incr();
- numVolumeUpdates.incr();
- }
-
- public void incNumVolumeInfos() {
- numVolumeOps.incr();
- numVolumeInfos.incr();
- }
-
- public void incNumVolumeDeletes() {
- numVolumeOps.incr();
- numVolumeDeletes.incr();
- }
-
- public void incNumVolumeCheckAccesses() {
- numVolumeOps.incr();
- numVolumeCheckAccesses.incr();
- }
-
- public void incNumBucketCreates() {
- numBucketOps.incr();
- numBucketCreates.incr();
- }
-
- public void incNumBucketInfos() {
- numBucketOps.incr();
- numBucketInfos.incr();
- }
-
- public void incNumBucketUpdates() {
- numBucketOps.incr();
- numBucketUpdates.incr();
- }
-
- public void incNumBucketDeletes() {
- numBucketOps.incr();
- numBucketDeletes.incr();
- }
-
- public void incNumBucketLists() {
- numBucketOps.incr();
- numBucketLists.incr();
- }
-
- public void incNumKeyLists() {
- numKeyOps.incr();
- numKeyLists.incr();
- }
-
- public void incNumVolumeLists() {
- numVolumeOps.incr();
- numVolumeLists.incr();
- }
-
- public void incNumGetServiceLists() {
- numGetServiceLists.incr();
- }
-
- public void incNumVolumeCreateFails() {
- numVolumeCreateFails.incr();
- }
-
- public void incNumVolumeUpdateFails() {
- numVolumeUpdateFails.incr();
- }
-
- public void incNumVolumeInfoFails() {
- numVolumeInfoFails.incr();
- }
-
- public void incNumVolumeDeleteFails() {
- numVolumeDeleteFails.incr();
- }
-
- public void incNumVolumeCheckAccessFails() {
- numVolumeCheckAccessFails.incr();
- }
-
- public void incNumBucketCreateFails() {
- numBucketCreateFails.incr();
- }
-
- public void incNumBucketInfoFails() {
- numBucketInfoFails.incr();
- }
-
- public void incNumBucketUpdateFails() {
- numBucketUpdateFails.incr();
- }
-
- public void incNumBucketDeleteFails() {
- numBucketDeleteFails.incr();
- }
-
- public void incNumKeyAllocates() {
- numKeyOps.incr();
- numKeyAllocate.incr();
- }
-
- public void incNumKeyAllocateFails() {
- numKeyAllocateFails.incr();
- }
-
- public void incNumKeyLookups() {
- numKeyOps.incr();
- numKeyLookup.incr();
- }
-
- public void incNumKeyLookupFails() {
- numKeyLookupFails.incr();
- }
-
- public void incNumKeyDeleteFails() {
- numKeyDeleteFails.incr();
- }
-
- public void incNumKeyDeletes() {
- numKeyOps.incr();
- numKeyDeletes.incr();
- }
-
- public void incNumKeyCommits() {
- numKeyOps.incr();
- numKeyCommits.incr();
- }
-
- public void incNumKeyCommitFails() {
- numKeyCommitFails.incr();
- }
-
- public void incNumBlockAllocateCalls() {
- numAllocateBlockCalls.incr();
- }
-
- public void incNumBlockAllocateCallFails() {
- numBlockAllocateCallFails.incr();
- }
-
- public void incNumBucketListFails() {
- numBucketListFails.incr();
- }
-
- public void incNumKeyListFails() {
- numKeyListFails.incr();
- }
-
- public void incNumVolumeListFails() {
- numVolumeListFails.incr();
- }
-
- public void incNumGetServiceListFails() {
- numGetServiceListFails.incr();
- }
-
- @VisibleForTesting
- public long getNumVolumeCreates() {
- return numVolumeCreates.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeUpdates() {
- return numVolumeUpdates.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeInfos() {
- return numVolumeInfos.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeDeletes() {
- return numVolumeDeletes.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeCheckAccesses() {
- return numVolumeCheckAccesses.value();
- }
-
- @VisibleForTesting
- public long getNumBucketCreates() {
- return numBucketCreates.value();
- }
-
- @VisibleForTesting
- public long getNumBucketInfos() {
- return numBucketInfos.value();
- }
-
- @VisibleForTesting
- public long getNumBucketUpdates() {
- return numBucketUpdates.value();
- }
-
- @VisibleForTesting
- public long getNumBucketDeletes() {
- return numBucketDeletes.value();
- }
-
- @VisibleForTesting
- public long getNumBucketLists() {
- return numBucketLists.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeLists() {
- return numVolumeLists.value();
- }
-
- @VisibleForTesting
- public long getNumKeyLists() {
- return numKeyLists.value();
- }
-
- @VisibleForTesting
- public long getNumGetServiceLists() {
- return numGetServiceLists.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeCreateFails() {
- return numVolumeCreateFails.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeUpdateFails() {
- return numVolumeUpdateFails.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeInfoFails() {
- return numVolumeInfoFails.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeDeleteFails() {
- return numVolumeDeleteFails.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeCheckAccessFails() {
- return numVolumeCheckAccessFails.value();
- }
-
- @VisibleForTesting
- public long getNumBucketCreateFails() {
- return numBucketCreateFails.value();
- }
-
- @VisibleForTesting
- public long getNumBucketInfoFails() {
- return numBucketInfoFails.value();
- }
-
- @VisibleForTesting
- public long getNumBucketUpdateFails() {
- return numBucketUpdateFails.value();
- }
-
- @VisibleForTesting
- public long getNumBucketDeleteFails() {
- return numBucketDeleteFails.value();
- }
-
- @VisibleForTesting
- public long getNumKeyAllocates() {
- return numKeyAllocate.value();
- }
-
- @VisibleForTesting
- public long getNumKeyAllocateFails() {
- return numKeyAllocateFails.value();
- }
-
- @VisibleForTesting
- public long getNumKeyLookups() {
- return numKeyLookup.value();
- }
-
- @VisibleForTesting
- public long getNumKeyLookupFails() {
- return numKeyLookupFails.value();
- }
-
- @VisibleForTesting
- public long getNumKeyDeletes() {
- return numKeyDeletes.value();
- }
-
- @VisibleForTesting
- public long getNumKeyDeletesFails() {
- return numKeyDeleteFails.value();
- }
-
- @VisibleForTesting
- public long getNumBucketListFails() {
- return numBucketListFails.value();
- }
-
- @VisibleForTesting
- public long getNumKeyListFails() {
- return numKeyListFails.value();
- }
-
- @VisibleForTesting
- public long getNumVolumeListFails() {
- return numVolumeListFails.value();
- }
-
- @VisibleForTesting
- public long getNumKeyCommits() {
- return numKeyCommits.value();
- }
-
- @VisibleForTesting
- public long getNumKeyCommitFails() {
- return numKeyCommitFails.value();
- }
-
- @VisibleForTesting
- public long getNumBlockAllocates() {
- return numAllocateBlockCalls.value();
- }
-
- @VisibleForTesting
- public long getNumBlockAllocateFails() {
- return numBlockAllocateCallFails.value();
- }
-
- @VisibleForTesting
- public long getNumGetServiceListFails() {
- return numGetServiceListFails.value();
- }
-
- public void unRegister() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- ms.unregisterSource(SOURCE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
deleted file mode 100644
index 8219212..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KSMStorage.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.ozone.scm.SCMStorage;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-/**
- * KSMStorage is responsible for management of the StorageDirectories used by
- * the KSM.
- */
-public class KSMStorage extends Storage {
-
- public static final String STORAGE_DIR = "ksm";
- public static final String KSM_ID = "ksmUuid";
-
- /**
- * Construct KSMStorage.
- * @throws IOException if any directories are inaccessible.
- */
- public KSMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.KSM, OzoneUtils.getOzoneMetaDirPath(conf), STORAGE_DIR);
- }
-
- public void setScmId(String scmId) throws IOException {
- if (getState() == StorageState.INITIALIZED) {
- throw new IOException("KSM is already initialized.");
- } else {
- getStorageInfo().setProperty(SCMStorage.SCM_ID, scmId);
- }
- }
-
- public void setKsmId(String ksmId) throws IOException {
- if (getState() == StorageState.INITIALIZED) {
- throw new IOException("KSM is already initialized.");
- } else {
- getStorageInfo().setProperty(KSM_ID, ksmId);
- }
- }
-
- /**
- * Retrieves the SCM ID from the version file.
- * @return SCM_ID
- */
- public String getScmId() {
- return getStorageInfo().getProperty(SCMStorage.SCM_ID);
- }
-
- /**
- * Retrieves the KSM ID from the version file.
- * @return KSM_ID
- */
- public String getKsmId() {
- return getStorageInfo().getProperty(KSM_ID);
- }
-
- @Override
- protected Properties getNodeProperties() {
- String ksmId = getKsmId();
- if (ksmId == null) {
- ksmId = UUID.randomUUID().toString();
- }
- Properties ksmProperties = new Properties();
- ksmProperties.setProperty(KSM_ID, ksmId);
- return ksmProperties;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
deleted file mode 100644
index 3beaed4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyDeletingService.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.BackgroundService;
-import org.apache.hadoop.utils.BackgroundTask;
-import org.apache.hadoop.utils.BackgroundTaskQueue;
-import org.apache.hadoop.utils.BackgroundTaskResult;
-import org.apache.hadoop.utils.BackgroundTaskResult.EmptyTaskResult;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT;
-
-/**
- * This is the background service to delete keys.
- * Scan the metadata of ksm periodically to get
- * the keys with prefix "#deleting" and ask scm to
- * delete metadata accordingly, if scm returns
- * success for keys, then clean up those keys.
- */
-public class KeyDeletingService extends BackgroundService {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(KeyDeletingService.class);
-
- // The thread pool size for key deleting service.
- private final static int KEY_DELETING_CORE_POOL_SIZE = 2;
-
- private final ScmBlockLocationProtocol scmClient;
- private final KeyManager manager;
- private final int keyLimitPerTask;
-
- public KeyDeletingService(ScmBlockLocationProtocol scmClient,
- KeyManager manager, long serviceInterval,
- long serviceTimeout, Configuration conf) {
- super("KeyDeletingService", serviceInterval, TimeUnit.MILLISECONDS,
- KEY_DELETING_CORE_POOL_SIZE, serviceTimeout);
- this.scmClient = scmClient;
- this.manager = manager;
- this.keyLimitPerTask = conf.getInt(OZONE_KEY_DELETING_LIMIT_PER_TASK,
- OZONE_KEY_DELETING_LIMIT_PER_TASK_DEFAULT);
- }
-
- @Override
- public BackgroundTaskQueue getTasks() {
- BackgroundTaskQueue queue = new BackgroundTaskQueue();
- queue.add(new KeyDeletingTask());
- return queue;
- }
-
- /**
- * A key deleting task scans KSM DB and looking for a certain number
- * of pending-deletion keys, sends these keys along with their associated
- * blocks to SCM for deletion. Once SCM confirms keys are deleted (once
- * SCM persisted the blocks info in its deletedBlockLog), it removes
- * these keys from the DB.
- */
- private class KeyDeletingTask implements
- BackgroundTask<BackgroundTaskResult> {
-
- @Override
- public int getPriority() {
- return 0;
- }
-
- @Override
- public BackgroundTaskResult call() throws Exception {
- try {
- long startTime = Time.monotonicNow();
- List<BlockGroup> keyBlocksList = manager
- .getPendingDeletionKeys(keyLimitPerTask);
- if (keyBlocksList.size() > 0) {
- LOG.info("Found {} to-delete keys in KSM", keyBlocksList.size());
- List<DeleteBlockGroupResult> results =
- scmClient.deleteKeyBlocks(keyBlocksList);
- for (DeleteBlockGroupResult result : results) {
- if (result.isSuccess()) {
- try {
- // Purge key from KSM DB.
- manager.deletePendingDeletionKey(result.getObjectKey());
- LOG.debug("Key {} deleted from KSM DB", result.getObjectKey());
- } catch (IOException e) {
- // if a pending deletion key is failed to delete,
- // print a warning here and retain it in this state,
- // so that it can be attempt to delete next time.
- LOG.warn("Failed to delete pending-deletion key {}",
- result.getObjectKey(), e);
- }
- } else {
- // Key deletion failed, retry in next interval.
- LOG.warn("Key {} deletion failed because some of the blocks"
- + " were failed to delete, failed blocks: {}",
- result.getObjectKey(),
- String.join(",", result.getFailedBlocks()));
- }
- }
-
- if (!results.isEmpty()) {
- LOG.info("Number of key deleted from KSM DB: {},"
- + " task elapsed time: {}ms",
- results.size(), Time.monotonicNow() - startTime);
- }
-
- return results::size;
- } else {
- LOG.debug("No pending deletion key found in KSM");
- }
- } catch (IOException e) {
- LOG.error("Unable to get pending deletion keys, retry in"
- + " next interval", e);
- }
- return EmptyTaskResult.newResult();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
deleted file mode 100644
index e71ce5f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/KeyManager.java
+++ /dev/null
@@ -1,165 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * Handles key level commands.
- */
-public interface KeyManager {
-
- /**
- * Start key manager.
- */
- void start();
-
- /**
- * Stop key manager.
- */
- void stop() throws IOException;
-
- /**
- * After calling commit, the key will be made visible. There can be multiple
- * open key writes in parallel (identified by client id). The most recently
- * committed one will be the one visible.
- *
- * @param args the key to commit.
- * @param clientID the client that is committing.
- * @throws IOException
- */
- void commitKey(KsmKeyArgs args, int clientID) throws IOException;
-
- /**
- * A client calls this on an open key, to request to allocate a new block,
- * and appended to the tail of current block list of the open client.
- *
- * @param args the key to append
- * @param clientID the client requesting block.
- * @return the reference to the new block.
- * @throws IOException
- */
- KsmKeyLocationInfo allocateBlock(KsmKeyArgs args, int clientID)
- throws IOException;
- /**
- * Given the args of a key to put, write an open key entry to meta data.
- *
- * In case that the container creation or key write failed on
- * DistributedStorageHandler, this key's metadata will still stay in KSM.
- * TODO garbage collect the open keys that never get closed
- *
- * @param args the args of the key provided by client.
- * @return a OpenKeySession instance client uses to talk to container.
- * @throws Exception
- */
- OpenKeySession openKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Look up an existing key. Return the info of the key to client side, which
- * DistributedStorageHandler will use to access the data on datanode.
- *
- * @param args the args of the key provided by client.
- * @return a KsmKeyInfo instance client uses to talk to container.
- * @throws IOException
- */
- KsmKeyInfo lookupKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Deletes an object by an object key. The key will be immediately removed
- * from KSM namespace and become invisible to clients. The object data
- * will be removed in async manner that might retain for some time.
- *
- * @param args the args of the key provided by client.
- * @throws IOException if specified key doesn't exist or
- * some other I/O errors while deleting an object.
- */
- void deleteKey(KsmKeyArgs args) throws IOException;
-
- /**
- * Returns a list of keys represented by {@link KsmKeyInfo}
- * in the given bucket.
- *
- * @param volumeName
- * the name of the volume.
- * @param bucketName
- * the name of the bucket.
- * @param startKey
- * the start key name, only the keys whose name is
- * after this value will be included in the result.
- * This key is excluded from the result.
- * @param keyPrefix
- * key name prefix, only the keys whose name has
- * this prefix will be included in the result.
- * @param maxKeys
- * the maximum number of keys to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of keys.
- * @throws IOException
- */
- List<KsmKeyInfo> listKeys(String volumeName,
- String bucketName, String startKey, String keyPrefix, int maxKeys)
- throws IOException;
-
- /**
- * Returns a list of pending deletion key info that ups to the given count.
- * Each entry is a {@link BlockGroup}, which contains the info about the
- * key name and all its associated block IDs. A pending deletion key is
- * stored with #deleting# prefix in KSM DB.
- *
- * @param count max number of keys to return.
- * @return a list of {@link BlockGroup} representing keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getPendingDeletionKeys(int count) throws IOException;
-
- /**
- * Deletes a pending deletion key by its name. This is often called when
- * key can be safely deleted from this layer. Once called, all footprints
- * of the key will be purged from KSM DB.
- *
- * @param objectKeyName object key name with #deleting# prefix.
- * @throws IOException if specified key doesn't exist or other I/O errors.
- */
- void deletePendingDeletionKey(String objectKeyName) throws IOException;
-
- /**
- * Returns a list of all still open key info. Which contains the info about
- * the key name and all its associated block IDs. A pending open key has
- * prefix #open# in KSM DB.
- *
- * @return a list of {@link BlockGroup} representing keys and blocks.
- * @throws IOException
- */
- List<BlockGroup> getExpiredOpenKeys() throws IOException;
-
- /**
- * Deletes a expired open key by its name. Called when a hanging key has been
- * lingering for too long. Once called, the open key entries gets removed
- * from KSM mdata data.
- *
- * @param objectKeyName object key name with #open# prefix.
- * @throws IOException if specified key doesn't exist or other I/O errors.
- */
- void deleteExpiredOpenKey(String objectKeyName) throws IOException;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
deleted file mode 100644
index 5069220..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientFactory.java
+++ /dev/null
@@ -1,308 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Proxy;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CLIENT_PROTOCOL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CLIENT_PROTOCOL_REST;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CLIENT_PROTOCOL_RPC;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_HTTP_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_HTTP_BIND_PORT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys.OZONE_KSM_PORT_DEFAULT;
-
-/**
- * Factory class to create different types of OzoneClients.
- * Based on <code>ozone.client.protocol</code>, it decides which
- * protocol to use for the communication.
- * Default value is
- * <code>org.apache.hadoop.ozone.client.rpc.RpcClient</code>.<br>
- * OzoneClientFactory constructs a proxy using
- * {@link OzoneClientInvocationHandler}
- * and creates OzoneClient instance with it.
- * {@link OzoneClientInvocationHandler} dispatches the call to
- * underlying {@link ClientProtocol} implementation.
- */
-public final class OzoneClientFactory {
-
- private static final Logger LOG = LoggerFactory.getLogger(
- OzoneClientFactory.class);
-
- /**
- * Private constructor, class is not meant to be initialized.
- */
- private OzoneClientFactory(){}
-
-
- /**
- * Constructs and return an OzoneClient with default configuration.
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getClient() throws IOException {
- LOG.info("Creating OzoneClient with default configuration.");
- return getClient(new OzoneConfiguration());
- }
-
- /**
- * Constructs and return an OzoneClient based on the configuration object.
- * Protocol type is decided by <code>ozone.client.protocol</code>.
- *
- * @param config
- * Configuration to be used for OzoneClient creation
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getClient(Configuration config)
- throws IOException {
- Preconditions.checkNotNull(config);
- Class<? extends ClientProtocol> clazz = (Class<? extends ClientProtocol>)
- config.getClass(OZONE_CLIENT_PROTOCOL, OZONE_CLIENT_PROTOCOL_RPC);
- return getClient(getClientProtocol(clazz, config), config);
- }
-
- /**
- * Returns an OzoneClient which will use RPC protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRpcClient(String ksmHost)
- throws IOException {
- return getRpcClient(ksmHost, OZONE_KSM_PORT_DEFAULT,
- new OzoneConfiguration());
- }
-
- /**
- * Returns an OzoneClient which will use RPC protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @param ksmRpcPort
- * RPC port of KeySpaceManager.
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort)
- throws IOException {
- return getRpcClient(ksmHost, ksmRpcPort, new OzoneConfiguration());
- }
-
- /**
- * Returns an OzoneClient which will use RPC protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @param ksmRpcPort
- * RPC port of KeySpaceManager.
- *
- * @param config
- * Configuration to be used for OzoneClient creation
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRpcClient(String ksmHost, Integer ksmRpcPort,
- Configuration config)
- throws IOException {
- Preconditions.checkNotNull(ksmHost);
- Preconditions.checkNotNull(ksmRpcPort);
- Preconditions.checkNotNull(config);
- config.set(OZONE_KSM_ADDRESS_KEY, ksmHost + ":" + ksmRpcPort);
- return getRpcClient(config);
- }
-
- /**
- * Returns an OzoneClient which will use RPC protocol.
- *
- * @param config
- * used for OzoneClient creation
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRpcClient(Configuration config)
- throws IOException {
- Preconditions.checkNotNull(config);
- return getClient(getClientProtocol(OZONE_CLIENT_PROTOCOL_RPC, config),
- config);
- }
-
- /**
- * Returns an OzoneClient which will use REST protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRestClient(String ksmHost)
- throws IOException {
- return getRestClient(ksmHost, OZONE_KSM_HTTP_BIND_PORT_DEFAULT);
- }
-
- /**
- * Returns an OzoneClient which will use REST protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @param ksmHttpPort
- * HTTP port of KeySpaceManager.
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort)
- throws IOException {
- return getRestClient(ksmHost, ksmHttpPort, new OzoneConfiguration());
- }
-
- /**
- * Returns an OzoneClient which will use REST protocol.
- *
- * @param ksmHost
- * hostname of KeySpaceManager to connect.
- *
- * @param ksmHttpPort
- * HTTP port of KeySpaceManager.
- *
- * @param config
- * Configuration to be used for OzoneClient creation
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRestClient(String ksmHost, Integer ksmHttpPort,
- Configuration config)
- throws IOException {
- Preconditions.checkNotNull(ksmHost);
- Preconditions.checkNotNull(ksmHttpPort);
- Preconditions.checkNotNull(config);
- config.set(OZONE_KSM_HTTP_ADDRESS_KEY, ksmHost + ":" + ksmHttpPort);
- return getRestClient(config);
- }
-
- /**
- * Returns an OzoneClient which will use REST protocol.
- *
- * @param config
- * Configuration to be used for OzoneClient creation
- *
- * @return OzoneClient
- *
- * @throws IOException
- */
- public static OzoneClient getRestClient(Configuration config)
- throws IOException {
- Preconditions.checkNotNull(config);
- return getClient(getClientProtocol(OZONE_CLIENT_PROTOCOL_REST, config),
- config);
- }
-
- /**
- * Creates OzoneClient with the given ClientProtocol and Configuration.
- *
- * @param clientProtocol
- * Protocol to be used by the OzoneClient
- *
- * @param config
- * Configuration to be used for OzoneClient creation
- */
- private static OzoneClient getClient(ClientProtocol clientProtocol,
- Configuration config) {
- OzoneClientInvocationHandler clientHandler =
- new OzoneClientInvocationHandler(clientProtocol);
- ClientProtocol proxy = (ClientProtocol) Proxy.newProxyInstance(
- OzoneClientInvocationHandler.class.getClassLoader(),
- new Class<?>[]{ClientProtocol.class}, clientHandler);
- return new OzoneClient(config, proxy);
- }
-
- /**
- * Returns an instance of Protocol class.
- *
- * @param protocolClass
- * Class object of the ClientProtocol.
- *
- * @param config
- * Configuration used to initialize ClientProtocol.
- *
- * @return ClientProtocol
- *
- * @throws IOException
- */
- private static ClientProtocol getClientProtocol(
- Class<? extends ClientProtocol> protocolClass, Configuration config)
- throws IOException {
- try {
- LOG.debug("Using {} as client protocol.",
- protocolClass.getCanonicalName());
- Constructor<? extends ClientProtocol> ctor =
- protocolClass.getConstructor(Configuration.class);
- return ctor.newInstance(config);
- } catch (Exception e) {
- final String message = "Couldn't create protocol " + protocolClass;
- LOG.error(message + " exception:" + e);
- if (e.getCause() instanceof IOException) {
- throw (IOException) e.getCause();
- } else if (e instanceof InvocationTargetException) {
- throw new IOException(message,
- ((InvocationTargetException) e).getTargetException());
- } else {
- throw new IOException(message, e);
- }
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
deleted file mode 100644
index 3051e2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientInvocationHandler.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.InvocationHandler;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-
-/**
- * Invocation Handler for ozone client which dispatches the call to underlying
- * ClientProtocol implementation.
- */
-public class OzoneClientInvocationHandler implements InvocationHandler {
-
-
- private static final Logger LOG = LoggerFactory.getLogger(OzoneClient.class);
- private final ClientProtocol target;
-
- /**
- * Constructs OzoneClientInvocationHandler with the proxy.
- * @param target proxy to be used for method invocation.
- */
- public OzoneClientInvocationHandler(ClientProtocol target) {
- this.target = target;
- }
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args)
- throws Throwable {
- LOG.trace("Invoking method {} on proxy {}", method, proxy);
- try {
- long startTime = Time.monotonicNow();
- Object result = method.invoke(target, args);
- LOG.debug("Call: {} took {} ms", method,
- Time.monotonicNow() - startTime);
- return result;
- } catch(InvocationTargetException iEx) {
- throw iEx.getCause();
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
deleted file mode 100644
index 1830bdf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneClientUtils.java
+++ /dev/null
@@ -1,874 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Optional;
-
-import com.google.common.base.Preconditions;
-import com.google.common.net.HostAndPort;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.http.client.config.RequestConfig;
-import org.apache.http.client.methods.HttpRequestBase;
-import org.apache.http.impl.client.CloseableHttpClient;
-import org.apache.http.impl.client.HttpClients;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.InetSocketAddress;
-import java.text.ParseException;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.time.ZonedDateTime;
-import java.time.format.DateTimeFormatter;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_SERVICERPC_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSI_PORT_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_ADDRESS_KEY;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_BIND_HOST_DEFAULT;
-import static org.apache.hadoop.ozone.ksm.KSMConfigKeys
- .OZONE_KSM_PORT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_DEADNODE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_DEADNODE_INTERVAL;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_INTERVAL;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
-
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_STALENODE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_STALENODE_INTERVAL;
-
-/**
- * Utility methods for Ozone and Container Clients.
- *
- * The methods to retrieve SCM service endpoints assume there is a single
- * SCM service instance. This will change when we switch to replicated service
- * instances for redundancy.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public final class OzoneClientUtils {
- private static final Logger LOG = LoggerFactory.getLogger(
- OzoneClientUtils.class);
- private static final int NO_PORT = -1;
-
- /**
- * Date format that used in ozone. Here the format is thread safe to use.
- */
- private static final ThreadLocal<DateTimeFormatter> DATE_FORMAT =
- ThreadLocal.withInitial(() -> {
- DateTimeFormatter format =
- DateTimeFormatter.ofPattern(OzoneConsts.OZONE_DATE_FORMAT);
- return format.withZone(ZoneId.of(OzoneConsts.OZONE_TIME_ZONE));
- });
-
- /**
- * The service ID of the solitary Ozone SCM service.
- */
- public static final String OZONE_SCM_SERVICE_ID = "OzoneScmService";
- public static final String OZONE_SCM_SERVICE_INSTANCE_ID =
- "OzoneScmServiceInstance";
-
- private OzoneClientUtils() {
- // Never constructed
- }
-
- /**
- * Retrieve the socket addresses of all storage container managers.
- *
- * @param conf
- * @return A collection of SCM addresses
- * @throws IllegalArgumentException If the configuration is invalid
- */
- public static Collection<InetSocketAddress> getSCMAddresses(
- Configuration conf) throws IllegalArgumentException {
- Collection<InetSocketAddress> addresses =
- new HashSet<InetSocketAddress>();
- Collection<String> names =
- conf.getTrimmedStringCollection(ScmConfigKeys.OZONE_SCM_NAMES);
- if (names == null || names.isEmpty()) {
- throw new IllegalArgumentException(ScmConfigKeys.OZONE_SCM_NAMES
- + " need to be a set of valid DNS names or IP addresses."
- + " Null or empty address list found.");
- }
-
- final com.google.common.base.Optional<Integer>
- defaultPort = com.google.common.base.Optional.of(ScmConfigKeys
- .OZONE_SCM_DEFAULT_PORT);
- for (String address : names) {
- com.google.common.base.Optional<String> hostname =
- OzoneClientUtils.getHostName(address);
- if (!hostname.isPresent()) {
- throw new IllegalArgumentException("Invalid hostname for SCM: "
- + hostname);
- }
- com.google.common.base.Optional<Integer> port =
- OzoneClientUtils.getHostPort(address);
- InetSocketAddress addr = NetUtils.createSocketAddr(hostname.get(),
- port.or(defaultPort.get()));
- addresses.add(addr);
- }
- return addresses;
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the SCM.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM client endpoint.
- */
- public static InetSocketAddress getScmAddressForClients(Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
- if (!host.isPresent()) {
- throw new IllegalArgumentException(
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
- " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
- " on configuring Ozone.");
- }
-
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(host.get() + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the SCM for block service. If
- * {@link ScmConfigKeys#OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY} is not defined
- * then {@link ScmConfigKeys#OZONE_SCM_CLIENT_ADDRESS_KEY} is used.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM block client endpoint.
- * @throws IllegalArgumentException if configuration is not defined.
- */
- public static InetSocketAddress getScmAddressForBlockClients(
- Configuration conf) {
- Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
- if (!host.isPresent()) {
- host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
- if (!host.isPresent()) {
- throw new IllegalArgumentException(
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY +
- " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration" +
- " for details on configuring Ozone.");
- }
- }
-
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(host.get() + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that should be used by DataNodes to connect
- * to the SCM.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM service endpoint.
- */
- public static InetSocketAddress getScmAddressForDataNodes(
- Configuration conf) {
- // We try the following settings in decreasing priority to retrieve the
- // target host.
- // - OZONE_SCM_DATANODE_ADDRESS_KEY
- // - OZONE_SCM_CLIENT_ADDRESS_KEY
- //
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
- if (!host.isPresent()) {
- throw new IllegalArgumentException(
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY +
- " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration for details" +
- " on configuring Ozone.");
- }
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
-
- InetSocketAddress addr = NetUtils.createSocketAddr(host.get() + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
-
- return addr;
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the SCM.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM client endpoint.
- */
- public static InetSocketAddress getScmClientBindAddress(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY);
-
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT) + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the SCM Block service.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM block client endpoint.
- */
- public static InetSocketAddress getScmBlockClientBindAddress(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_KEY);
-
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_BIND_HOST_DEFAULT) +
- ":" + port.or(ScmConfigKeys.OZONE_SCM_BLOCK_CLIENT_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that should be used by DataNodes to connect
- * to the SCM.
- *
- * @param conf
- * @return Target InetSocketAddress for the SCM service endpoint.
- */
- public static InetSocketAddress getScmDataNodeBindAddress(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_KEY);
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(ScmConfigKeys.OZONE_SCM_DATANODE_BIND_HOST_DEFAULT) + ":" +
- port.or(ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT));
- }
-
-
- /**
- * Retrieve the socket address that is used by KSM.
- * @param conf
- * @return Target InetSocketAddress for the SCM service endpoint.
- */
- public static InetSocketAddress getKsmAddress(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- OZONE_KSM_ADDRESS_KEY);
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- OZONE_KSM_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(OZONE_KSM_BIND_HOST_DEFAULT) + ":" +
- port.or(OZONE_KSM_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to KSM.
- * @param conf
- * @return Target InetSocketAddress for the KSM service endpoint.
- */
- public static InetSocketAddress getKsmAddressForClients(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- OZONE_KSM_ADDRESS_KEY);
-
- if (!host.isPresent()) {
- throw new IllegalArgumentException(
- OZONE_KSM_ADDRESS_KEY + " must be defined. See" +
- " https://wiki.apache.org/hadoop/Ozone#Configuration for" +
- " details on configuring Ozone.");
- }
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- OZONE_KSM_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.get() + ":" + port.or(OZONE_KSM_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that is used by CBlock Service.
- * @param conf
- * @return Target InetSocketAddress for the CBlock Service endpoint.
- */
- public static InetSocketAddress getCblockServiceRpcAddr(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- DFS_CBLOCK_SERVICERPC_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" +
- port.or(DFS_CBLOCK_SERVICERPC_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the socket address that is used by CBlock Server.
- * @param conf
- * @return Target InetSocketAddress for the CBlock Server endpoint.
- */
- public static InetSocketAddress getCblockServerRpcAddr(
- Configuration conf) {
- final Optional<String> host = getHostNameFromConfigKeys(conf,
- DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
-
- // If no port number is specified then we'll just try the defaultBindPort.
- final Optional<Integer> port = getPortNumberFromConfigKeys(conf,
- DFS_CBLOCK_JSCSIRPC_ADDRESS_KEY);
-
- return NetUtils.createSocketAddr(
- host.or(DFS_CBLOCK_SERVICERPC_HOSTNAME_DEFAULT) + ":" +
- port.or(DFS_CBLOCK_JSCSI_PORT_DEFAULT));
- }
-
- /**
- * Retrieve the hostname, trying the supplied config keys in order.
- * Each config value may be absent, or if present in the format
- * host:port (the :port part is optional).
- *
- * @param conf - Conf
- * @param keys a list of configuration key names.
- *
- * @return first hostname component found from the given keys, or absent.
- * @throws IllegalArgumentException if any values are not in the 'host'
- * or host:port format.
- */
- public static Optional<String> getHostNameFromConfigKeys(Configuration conf,
- String... keys) {
- for (final String key : keys) {
- final String value = conf.getTrimmed(key);
- final Optional<String> hostName = getHostName(value);
- if (hostName.isPresent()) {
- return hostName;
- }
- }
- return Optional.absent();
- }
-
- /**
- * Gets the hostname or Indicates that it is absent.
- * @param value host or host:port
- * @return hostname
- */
- public static Optional<String> getHostName(String value) {
- if ((value == null) || value.isEmpty()) {
- return Optional.absent();
- }
- return Optional.of(HostAndPort.fromString(value).getHostText());
- }
-
- /**
- * Gets the port if there is one, throws otherwise.
- * @param value String in host:port format.
- * @return Port
- */
- public static Optional<Integer> getHostPort(String value) {
- if((value == null) || value.isEmpty()) {
- return Optional.absent();
- }
- int port = HostAndPort.fromString(value).getPortOrDefault(NO_PORT);
- if (port == NO_PORT) {
- return Optional.absent();
- } else {
- return Optional.of(port);
- }
- }
-
- /**
- * Returns the cache value to be used for list calls.
- * @param conf Configuration object
- * @return list cache size
- */
- public static int getListCacheSize(Configuration conf) {
- return conf.getInt(OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE,
- OzoneConfigKeys.OZONE_CLIENT_LIST_CACHE_SIZE_DEFAULT);
- }
-
- /**
- * Retrieve the port number, trying the supplied config keys in order.
- * Each config value may be absent, or if present in the format
- * host:port (the :port part is optional).
- *
- * @param conf Conf
- * @param keys a list of configuration key names.
- *
- * @return first port number component found from the given keys, or absent.
- * @throws IllegalArgumentException if any values are not in the 'host'
- * or host:port format.
- */
- public static Optional<Integer> getPortNumberFromConfigKeys(
- Configuration conf, String... keys) {
- for (final String key : keys) {
- final String value = conf.getTrimmed(key);
- final Optional<Integer> hostPort = getHostPort(value);
- if (hostPort.isPresent()) {
- return hostPort;
- }
- }
- return Optional.absent();
- }
-
- /**
- * Return the list of service addresses for the Ozone SCM. This method is used
- * by the DataNodes to determine the service instances to connect to.
- *
- * @param conf
- * @return list of SCM service addresses.
- */
- public static Map<String, ? extends Map<String, InetSocketAddress>>
- getScmServiceRpcAddresses(Configuration conf) {
- final Map<String, InetSocketAddress> serviceInstances = new HashMap<>();
- serviceInstances.put(OZONE_SCM_SERVICE_INSTANCE_ID,
- getScmAddressForDataNodes(conf));
-
- final Map<String, Map<String, InetSocketAddress>> services =
- new HashMap<>();
- services.put(OZONE_SCM_SERVICE_ID, serviceInstances);
- return services;
- }
-
- /**
- * Checks that a given value is with a range.
- *
- * For example, sanitizeUserArgs(17, 3, 5, 10)
- * ensures that 17 is greater/equal than 3 * 5 and less/equal to 3 * 10.
- *
- * @param valueTocheck - value to check
- * @param baseValue - the base value that is being used.
- * @param minFactor - range min - a 2 here makes us ensure that value
- * valueTocheck is at least twice the baseValue.
- * @param maxFactor - range max
- * @return long
- */
- private static long sanitizeUserArgs(long valueTocheck, long baseValue,
- long minFactor, long maxFactor)
- throws IllegalArgumentException {
- if ((valueTocheck >= (baseValue * minFactor)) &&
- (valueTocheck <= (baseValue * maxFactor))) {
- return valueTocheck;
- }
- String errMsg = String.format("%d is not within min = %d or max = " +
- "%d", valueTocheck, baseValue * minFactor, baseValue * maxFactor);
- throw new IllegalArgumentException(errMsg);
- }
-
- /**
- * Returns the interval in which the heartbeat processor thread runs.
- *
- * @param conf - Configuration
- * @return long in Milliseconds.
- */
- public static long getScmheartbeatCheckerInterval(Configuration conf) {
- return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
- ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
- }
-
- /**
- * Heartbeat Interval - Defines the heartbeat frequency from a datanode to
- * SCM.
- *
- * @param conf - Ozone Config
- * @return - HB interval in seconds.
- */
- public static long getScmHeartbeatInterval(Configuration conf) {
- return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_INTERVAL,
- ScmConfigKeys.OZONE_SCM_HEARBEAT_INTERVAL_DEFAULT,
- TimeUnit.SECONDS);
- }
-
- /**
- * Get the Stale Node interval, which is used by SCM to flag a datanode as
- * stale, if the heartbeat from that node has been missing for this duration.
- *
- * @param conf - Configuration.
- * @return - Long, Milliseconds to wait before flagging a node as stale.
- */
- public static long getStaleNodeInterval(Configuration conf) {
-
- long staleNodeIntervalMs =
- conf.getTimeDuration(OZONE_SCM_STALENODE_INTERVAL,
- OZONE_SCM_STALENODE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
-
- long heartbeatThreadFrequencyMs = getScmheartbeatCheckerInterval(conf);
-
- long heartbeatIntervalMs = getScmHeartbeatInterval(conf) * 1000;
-
-
- // Make sure that StaleNodeInterval is configured way above the frequency
- // at which we run the heartbeat thread.
- //
- // Here we check that staleNodeInterval is at least five times more than the
- // frequency at which the accounting thread is going to run.
- try {
- sanitizeUserArgs(staleNodeIntervalMs, heartbeatThreadFrequencyMs,
- 5, 1000);
- } catch (IllegalArgumentException ex) {
- LOG.error("Stale Node Interval is cannot be honored due to " +
- "mis-configured {}. ex: {}",
- OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, ex);
- throw ex;
- }
-
- // Make sure that stale node value is greater than configured value that
- // datanodes are going to send HBs.
- try {
- sanitizeUserArgs(staleNodeIntervalMs, heartbeatIntervalMs, 3, 1000);
- } catch (IllegalArgumentException ex) {
- LOG.error("Stale Node Interval MS is cannot be honored due to " +
- "mis-configured {}. ex: {}", OZONE_SCM_HEARTBEAT_INTERVAL, ex);
- throw ex;
- }
- return staleNodeIntervalMs;
- }
-
- /**
- * Gets the interval for dead node flagging. This has to be a value that is
- * greater than stale node value, and by transitive relation we also know
- * that this value is greater than heartbeat interval and heartbeatProcess
- * Interval.
- *
- * @param conf - Configuration.
- * @return - the interval for dead node flagging.
- */
- public static long getDeadNodeInterval(Configuration conf) {
- long staleNodeIntervalMs = getStaleNodeInterval(conf);
- long deadNodeIntervalMs = conf.getTimeDuration(OZONE_SCM_DEADNODE_INTERVAL,
- OZONE_SCM_DEADNODE_INTERVAL_DEFAULT,
- TimeUnit.MILLISECONDS);
-
- try {
- // Make sure that dead nodes Ms is at least twice the time for staleNodes
- // with a max of 1000 times the staleNodes.
- sanitizeUserArgs(deadNodeIntervalMs, staleNodeIntervalMs, 2, 1000);
- } catch (IllegalArgumentException ex) {
- LOG.error("Dead Node Interval MS is cannot be honored due to " +
- "mis-configured {}. ex: {}", OZONE_SCM_STALENODE_INTERVAL, ex);
- throw ex;
- }
- return deadNodeIntervalMs;
- }
-
- /**
- * Returns the maximum number of heartbeat to process per loop of the process
- * thread.
- * @param conf Configuration
- * @return - int -- Number of HBs to process
- */
- public static int getMaxHBToProcessPerLoop(Configuration conf) {
- return conf.getInt(ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS,
- ScmConfigKeys.OZONE_SCM_MAX_HB_COUNT_TO_PROCESS_DEFAULT);
- }
-
- /**
- * Timeout value for the RPC from Datanode to SCM, primarily used for
- * Heartbeats and container reports.
- *
- * @param conf - Ozone Config
- * @return - Rpc timeout in Milliseconds.
- */
- public static long getScmRpcTimeOutInMilliseconds(Configuration conf) {
- return conf.getTimeDuration(OZONE_SCM_HEARTBEAT_RPC_TIMEOUT,
- OZONE_SCM_HEARTBEAT_RPC_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
- }
-
- /**
- * Log Warn interval.
- *
- * @param conf - Ozone Config
- * @return - Log warn interval.
- */
- public static int getLogWarnInterval(Configuration conf) {
- return conf.getInt(OZONE_SCM_HEARTBEAT_LOG_WARN_INTERVAL_COUNT,
- OZONE_SCM_HEARTBEAT_LOG_WARN_DEFAULT);
- }
-
- /**
- * returns the Container port.
- * @param conf - Conf
- * @return port number.
- */
- public static int getContainerPort(Configuration conf) {
- return conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
- }
-
- /**
- * After starting an RPC server, updates configuration with the actual
- * listening address of that server. The listening address may be different
- * from the configured address if, for example, the configured address uses
- * port 0 to request use of an ephemeral port.
- *
- * @param conf configuration to update
- * @param rpcAddressKey configuration key for RPC server address
- * @param addr configured address
- * @param rpcServer started RPC server.
- */
- public static InetSocketAddress updateRPCListenAddress(
- OzoneConfiguration conf, String rpcAddressKey,
- InetSocketAddress addr, RPC.Server rpcServer) {
- return updateListenAddress(conf, rpcAddressKey, addr,
- rpcServer.getListenerAddress());
- }
-
- /**
- * After starting an server, updates configuration with the actual
- * listening address of that server. The listening address may be different
- * from the configured address if, for example, the configured address uses
- * port 0 to request use of an ephemeral port.
- *
- * @param conf configuration to update
- * @param addressKey configuration key for RPC server address
- * @param addr configured address
- * @param listenAddr the real listening address.
- */
- public static InetSocketAddress updateListenAddress(OzoneConfiguration conf,
- String addressKey, InetSocketAddress addr, InetSocketAddress listenAddr) {
- InetSocketAddress updatedAddr = new InetSocketAddress(addr.getHostString(),
- listenAddr.getPort());
- conf.set(addressKey,
- addr.getHostString() + ":" + listenAddr.getPort());
- return updatedAddr;
- }
-
- /**
- * Releases a http connection if the request is not null.
- * @param request
- */
- public static void releaseConnection(HttpRequestBase request) {
- if (request != null) {
- request.releaseConnection();
- }
- }
-
- /**
- * @return a default instance of {@link CloseableHttpClient}.
- */
- public static CloseableHttpClient newHttpClient() {
- return OzoneClientUtils.newHttpClient(new OzoneConfiguration());
- }
-
- /**
- * Returns a {@link CloseableHttpClient} configured by given configuration.
- * If conf is null, returns a default instance.
- *
- * @param conf configuration
- * @return a {@link CloseableHttpClient} instance.
- */
- public static CloseableHttpClient newHttpClient(Configuration conf) {
- long socketTimeout = OzoneConfigKeys
- .OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT;
- long connectionTimeout = OzoneConfigKeys
- .OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT;
- if (conf != null) {
- socketTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_SOCKET_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- connectionTimeout = conf.getTimeDuration(
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT,
- OzoneConfigKeys.OZONE_CLIENT_CONNECTION_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- }
-
- CloseableHttpClient client = HttpClients.custom()
- .setDefaultRequestConfig(
- RequestConfig.custom()
- .setSocketTimeout(Math.toIntExact(socketTimeout))
- .setConnectTimeout(Math.toIntExact(connectionTimeout))
- .build())
- .build();
- return client;
- }
-
- /**
- * verifies that bucket name / volume name is a valid DNS name.
- *
- * @param resName Bucket or volume Name to be validated
- *
- * @throws IllegalArgumentException
- */
- public static void verifyResourceName(String resName)
- throws IllegalArgumentException {
-
- if (resName == null) {
- throw new IllegalArgumentException("Bucket or Volume name is null");
- }
-
- if ((resName.length() < OzoneConsts.OZONE_MIN_BUCKET_NAME_LENGTH) ||
- (resName.length() > OzoneConsts.OZONE_MAX_BUCKET_NAME_LENGTH)) {
- throw new IllegalArgumentException(
- "Bucket or Volume length is illegal, " +
- "valid length is 3-63 characters");
- }
-
- if ((resName.charAt(0) == '.') || (resName.charAt(0) == '-')) {
- throw new IllegalArgumentException(
- "Bucket or Volume name cannot start with a period or dash");
- }
-
- if ((resName.charAt(resName.length() - 1) == '.') ||
- (resName.charAt(resName.length() - 1) == '-')) {
- throw new IllegalArgumentException(
- "Bucket or Volume name cannot end with a period or dash");
- }
-
- boolean isIPv4 = true;
- char prev = (char) 0;
-
- for (int index = 0; index < resName.length(); index++) {
- char currChar = resName.charAt(index);
-
- if (currChar != '.') {
- isIPv4 = ((currChar >= '0') && (currChar <= '9')) && isIPv4;
- }
-
- if (currChar > 'A' && currChar < 'Z') {
- throw new IllegalArgumentException(
- "Bucket or Volume name does not support uppercase characters");
- }
-
- if ((currChar != '.') && (currChar != '-')) {
- if ((currChar < '0') || (currChar > '9' && currChar < 'a') ||
- (currChar > 'z')) {
- throw new IllegalArgumentException("Bucket or Volume name has an " +
- "unsupported character : " +
- currChar);
- }
- }
-
- if ((prev == '.') && (currChar == '.')) {
- throw new IllegalArgumentException("Bucket or Volume name should not " +
- "have two contiguous periods");
- }
-
- if ((prev == '-') && (currChar == '.')) {
- throw new IllegalArgumentException(
- "Bucket or Volume name should not have period after dash");
- }
-
- if ((prev == '.') && (currChar == '-')) {
- throw new IllegalArgumentException(
- "Bucket or Volume name should not have dash after period");
- }
- prev = currChar;
- }
-
- if (isIPv4) {
- throw new IllegalArgumentException(
- "Bucket or Volume name cannot be an IPv4 address or all numeric");
- }
- }
-
- /**
- * Convert time in millisecond to a human readable format required in ozone.
- * @return a human readable string for the input time
- */
- public static String formatDateTime(long millis) {
- ZonedDateTime dateTime = ZonedDateTime.ofInstant(
- Instant.ofEpochSecond(millis), DATE_FORMAT.get().getZone());
- return DATE_FORMAT.get().format(dateTime);
- }
-
- /**
- * Convert time in ozone date format to millisecond.
- * @return time in milliseconds
- */
- public static long formatDateTime(String date) throws ParseException {
- Preconditions.checkNotNull(date, "Date string should not be null.");
- return ZonedDateTime.parse(date, DATE_FORMAT.get())
- .toInstant().getEpochSecond();
- }
-
- /**
- * Returns the maximum no of outstanding async requests to be handled by
- * Standalone and Ratis client.
- */
- public static int getMaxOutstandingRequests(Configuration config) {
- return config
- .getInt(ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS,
- ScmConfigKeys.SCM_CONTAINER_CLIENT_MAX_OUTSTANDING_REQUESTS_DEFAULT);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
deleted file mode 100644
index 0c723dd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneKey.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * A class that encapsulates OzoneKey.
- */
-public class OzoneKey {
-
- /**
- * Name of the Volume the Key belongs to.
- */
- private final String volumeName;
- /**
- * Name of the Bucket the Key belongs to.
- */
- private final String bucketName;
- /**
- * Name of the Key.
- */
- private final String name;
- /**
- * Size of the data.
- */
- private final long dataSize;
- /**
- * Creation time of the key.
- */
- private long creationTime;
- /**
- * Modification time of the key.
- */
- private long modificationTime;
-
- /**
- * Constructs OzoneKey from KsmKeyInfo.
- *
- */
- public OzoneKey(String volumeName, String bucketName,
- String keyName, long size, long creationTime,
- long modificationTime) {
- this.volumeName = volumeName;
- this.bucketName = bucketName;
- this.name = keyName;
- this.dataSize = size;
- this.creationTime = creationTime;
- this.modificationTime = modificationTime;
- }
-
- /**
- * Returns Volume Name associated with the Key.
- *
- * @return volumeName
- */
- public String getVolumeName() {
- return volumeName;
- }
-
- /**
- * Returns Bucket Name associated with the Key.
- *
- * @return bucketName
- */
- public String getBucketName(){
- return bucketName;
- }
-
- /**
- * Returns the Key Name.
- *
- * @return keyName
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns the size of the data.
- *
- * @return dataSize
- */
- public long getDataSize() {
- return dataSize;
- }
-
- /**
- * Returns the creation time of the key.
- *
- * @return creation time
- */
- public long getCreationTime() {
- return creationTime;
- }
-
- /**
- * Returns the modification time of the key.
- *
- * @return modification time
- */
- public long getModificationTime() {
- return modificationTime;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
deleted file mode 100644
index 032dd60..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneQuota.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.OzoneConsts;
-
-
-/**
- * represents an OzoneQuota Object that can be applied to
- * a storage volume.
- */
-public class OzoneQuota {
-
- public static final String OZONE_QUOTA_BYTES = "BYTES";
- public static final String OZONE_QUOTA_MB = "MB";
- public static final String OZONE_QUOTA_GB = "GB";
- public static final String OZONE_QUOTA_TB = "TB";
-
- private Units unit;
- private long size;
-
- /** Quota Units.*/
- public enum Units {UNDEFINED, BYTES, KB, MB, GB, TB}
-
- /**
- * Returns size.
- *
- * @return long
- */
- public long getSize() {
- return size;
- }
-
- /**
- * Returns Units.
- *
- * @return Unit in MB, GB or TB
- */
- public Units getUnit() {
- return unit;
- }
-
- /**
- * Constructs a default Quota object.
- */
- public OzoneQuota() {
- this.size = 0;
- this.unit = Units.UNDEFINED;
- }
-
- /**
- * Constructor for Ozone Quota.
- *
- * @param size Long Size
- * @param unit MB, GB or TB
- */
- public OzoneQuota(long size, Units unit) {
- this.size = size;
- this.unit = unit;
- }
-
- /**
- * Formats a quota as a string.
- *
- * @param quota the quota to format
- * @return string representation of quota
- */
- public static String formatQuota(OzoneQuota quota) {
- return String.valueOf(quota.size) + quota.unit;
- }
-
- /**
- * Parses a user provided string and returns the
- * Quota Object.
- *
- * @param quotaString Quota String
- *
- * @return OzoneQuota object
- *
- * @throws IllegalArgumentException
- */
- public static OzoneQuota parseQuota(String quotaString)
- throws IllegalArgumentException {
-
- if ((quotaString == null) || (quotaString.isEmpty())) {
- throw new IllegalArgumentException(
- "Quota string cannot be null or empty.");
- }
-
- String uppercase = quotaString.toUpperCase().replaceAll("\\s+", "");
- String size = "";
- int nSize;
- Units currUnit = Units.MB;
- Boolean found = false;
- if (uppercase.endsWith(OZONE_QUOTA_MB)) {
- size = uppercase
- .substring(0, uppercase.length() - OZONE_QUOTA_MB.length());
- currUnit = Units.MB;
- found = true;
- }
-
- if (uppercase.endsWith(OZONE_QUOTA_GB)) {
- size = uppercase
- .substring(0, uppercase.length() - OZONE_QUOTA_GB.length());
- currUnit = Units.GB;
- found = true;
- }
-
- if (uppercase.endsWith(OZONE_QUOTA_TB)) {
- size = uppercase
- .substring(0, uppercase.length() - OZONE_QUOTA_TB.length());
- currUnit = Units.TB;
- found = true;
- }
-
- if (uppercase.endsWith(OZONE_QUOTA_BYTES)) {
- size = uppercase
- .substring(0, uppercase.length() - OZONE_QUOTA_BYTES.length());
- currUnit = Units.BYTES;
- found = true;
- }
-
- if (!found) {
- throw new IllegalArgumentException(
- "Quota unit not recognized. Supported values are BYTES, MB, GB and " +
- "TB.");
- }
-
- nSize = Integer.parseInt(size);
- if (nSize < 0) {
- throw new IllegalArgumentException("Quota cannot be negative.");
- }
-
- return new OzoneQuota(nSize, currUnit);
- }
-
-
- /**
- * Returns size in Bytes or -1 if there is no Quota.
- */
- public long sizeInBytes() {
- switch (this.unit) {
- case BYTES:
- return this.getSize();
- case MB:
- return this.getSize() * OzoneConsts.MB;
- case GB:
- return this.getSize() * OzoneConsts.GB;
- case TB:
- return this.getSize() * OzoneConsts.TB;
- case UNDEFINED:
- default:
- return -1;
- }
- }
-
- /**
- * Returns OzoneQuota corresponding to size in bytes.
- *
- * @param sizeInBytes size in bytes to be converted
- *
- * @return OzoneQuota object
- */
- public static OzoneQuota getOzoneQuota(long sizeInBytes) {
- long size;
- Units unit;
- if (sizeInBytes % OzoneConsts.TB == 0) {
- size = sizeInBytes / OzoneConsts.TB;
- unit = Units.TB;
- } else if (sizeInBytes % OzoneConsts.GB == 0) {
- size = sizeInBytes / OzoneConsts.GB;
- unit = Units.GB;
- } else if (sizeInBytes % OzoneConsts.MB == 0) {
- size = sizeInBytes / OzoneConsts.MB;
- unit = Units.MB;
- } else {
- size = sizeInBytes;
- unit = Units.BYTES;
- }
- return new OzoneQuota((int)size, unit);
- }
-
- @Override
- public String toString() {
- return size + " " + unit;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
deleted file mode 100644
index 31cbb40..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/OzoneVolume.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-import java.util.NoSuchElementException;
-
-/**
- * A class that encapsulates OzoneVolume.
- */
-public class OzoneVolume {
-
- /**
- * The proxy used for connecting to the cluster and perform
- * client operations.
- */
- private final ClientProtocol proxy;
-
- /**
- * Name of the Volume.
- */
- private final String name;
-
- /**
- * Admin Name of the Volume.
- */
- private String admin;
- /**
- * Owner of the Volume.
- */
- private String owner;
- /**
- * Quota allocated for the Volume.
- */
- private long quotaInBytes;
- /**
- * Creation time of the volume.
- */
- private long creationTime;
- /**
- * Volume ACLs.
- */
- private List<OzoneAcl> acls;
-
- private int listCacheSize;
-
- /**
- * Constructs OzoneVolume instance.
- * @param conf Configuration object.
- * @param proxy ClientProtocol proxy.
- * @param name Name of the volume.
- * @param admin Volume admin.
- * @param owner Volume owner.
- * @param quotaInBytes Volume quota in bytes.
- * @param creationTime creation time of the volume
- * @param acls ACLs associated with the volume.
- */
- public OzoneVolume(Configuration conf, ClientProtocol proxy, String name,
- String admin, String owner, long quotaInBytes,
- long creationTime, List<OzoneAcl> acls) {
- this.proxy = proxy;
- this.name = name;
- this.admin = admin;
- this.owner = owner;
- this.quotaInBytes = quotaInBytes;
- this.creationTime = creationTime;
- this.acls = acls;
- this.listCacheSize = OzoneClientUtils.getListCacheSize(conf);
- }
-
- /**
- * Returns Volume name.
- *
- * @return volumeName
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns Volume's admin name.
- *
- * @return adminName
- */
- public String getAdmin() {
- return admin;
- }
-
- /**
- * Returns Volume's owner name.
- *
- * @return ownerName
- */
- public String getOwner() {
- return owner;
- }
-
- /**
- * Returns Quota allocated for the Volume in bytes.
- *
- * @return quotaInBytes
- */
- public long getQuota() {
- return quotaInBytes;
- }
-
- /**
- * Returns creation time of the volume.
- *
- * @return creation time.
- */
- public long getCreationTime() {
- return creationTime;
- }
-
- /**
- * Returns OzoneAcl list associated with the Volume.
- *
- * @return aclMap
- */
- public List<OzoneAcl> getAcls() {
- return acls;
- }
-
- /**
- * Sets/Changes the owner of this Volume.
- * @param owner new owner
- * @throws IOException
- */
- public void setOwner(String owner) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(owner);
- proxy.setVolumeOwner(name, owner);
- this.owner = owner;
- }
-
- /**
- * Sets/Changes the quota of this Volume.
- * @param quota new quota
- * @throws IOException
- */
- public void setQuota(OzoneQuota quota) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(quota);
- proxy.setVolumeQuota(name, quota);
- this.quotaInBytes = quota.sizeInBytes();
- }
-
- /**
- * Creates a new Bucket in this Volume, with default values.
- * @param bucketName Name of the Bucket
- * @throws IOException
- */
- public void createBucket(String bucketName)
- throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(bucketName);
- OzoneClientUtils.verifyResourceName(bucketName);
- proxy.createBucket(name, bucketName);
- }
-
- /**
- * Creates a new Bucket in this Volume, with properties set in bucketArgs.
- * @param bucketName Name of the Bucket
- * @param bucketArgs Properties to be set
- * @throws IOException
- */
- public void createBucket(String bucketName, BucketArgs bucketArgs)
- throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(bucketName);
- Preconditions.checkNotNull(bucketArgs);
- OzoneClientUtils.verifyResourceName(bucketName);
- proxy.createBucket(name, bucketName, bucketArgs);
- }
-
- /**
- * Get the Bucket from this Volume.
- * @param bucketName Name of the Bucket
- * @return OzoneBucket
- * @throws IOException
- */
- public OzoneBucket getBucket(String bucketName) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(bucketName);
- OzoneClientUtils.verifyResourceName(bucketName);
- OzoneBucket bucket = proxy.getBucketDetails(name, bucketName);
- return bucket;
- }
-
- /**
- * Returns Iterator to iterate over all buckets in the volume.
- * The result can be restricted using bucket prefix, will return all
- * buckets if bucket prefix is null.
- *
- * @param bucketPrefix Bucket prefix to match
- * @return {@code Iterator<OzoneBucket>}
- */
- public Iterator<OzoneBucket> listBuckets(String bucketPrefix) {
- return new BucketIterator(bucketPrefix);
- }
-
- /**
- * Deletes the Bucket from this Volume.
- * @param bucketName Name of the Bucket
- * @throws IOException
- */
- public void deleteBucket(String bucketName) throws IOException {
- Preconditions.checkNotNull(proxy, "Client proxy is not set.");
- Preconditions.checkNotNull(bucketName);
- OzoneClientUtils.verifyResourceName(bucketName);
- proxy.deleteBucket(name, bucketName);
- }
-
-
- /**
- * An Iterator to iterate over {@link OzoneBucket} list.
- */
- private class BucketIterator implements Iterator<OzoneBucket> {
-
- private String bucketPrefix = null;
-
- private Iterator<OzoneBucket> currentIterator;
- private OzoneBucket currentValue;
-
-
- /**
- * Creates an Iterator to iterate over all buckets in the volume,
- * which matches volume prefix.
- * @param bucketPrefix
- */
- BucketIterator(String bucketPrefix) {
- this.bucketPrefix = bucketPrefix;
- this.currentValue = null;
- this.currentIterator = getNextListOfBuckets(null).iterator();
- }
-
- @Override
- public boolean hasNext() {
- if(!currentIterator.hasNext()) {
- currentIterator = getNextListOfBuckets(
- currentValue != null ? currentValue.getName() : null)
- .iterator();
- }
- return currentIterator.hasNext();
- }
-
- @Override
- public OzoneBucket next() {
- if(hasNext()) {
- currentValue = currentIterator.next();
- return currentValue;
- }
- throw new NoSuchElementException();
- }
-
- /**
- * Gets the next set of bucket list using proxy.
- * @param prevBucket
- * @return {@code List<OzoneVolume>}
- */
- private List<OzoneBucket> getNextListOfBuckets(String prevBucket) {
- try {
- return proxy.listBuckets(name, bucketPrefix, prevBucket, listCacheSize);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
deleted file mode 100644
index 971cfec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationFactor.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * The replication factor to be used while writing key into ozone.
- */
-public enum ReplicationFactor {
- ONE(1),
- THREE(3);
-
- /**
- * Integer representation of replication.
- */
- private int value;
-
- /**
- * Initializes ReplicationFactor with value.
- * @param value replication value
- */
- ReplicationFactor(int value) {
- this.value = value;
- }
-
- /**
- * Returns enum value corresponding to the int value.
- * @param value replication value
- * @return ReplicationFactor
- */
- public static ReplicationFactor valueOf(int value) {
- if(value == 1) {
- return ONE;
- }
- if (value == 3) {
- return THREE;
- }
- throw new IllegalArgumentException("Unsupported value: " + value);
- }
-
- /**
- * Returns integer representation of ReplicationFactor.
- * @return replication value
- */
- public int getValue() {
- return value;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
deleted file mode 100644
index 537c336..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/ReplicationType.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-/**
- * The replication type to be used while writing key into ozone.
- */
-public enum ReplicationType {
- RATIS,
- STAND_ALONE,
- CHAINED
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
deleted file mode 100644
index f1aa031..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/VolumeArgs.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.client;
-
-import org.apache.hadoop.ozone.OzoneAcl;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This class encapsulates the arguments that are
- * required for creating a volume.
- */
-public final class VolumeArgs {
-
- private final String admin;
- private final String owner;
- private final String quota;
- private final List<OzoneAcl> acls;
-
- /**
- * Private constructor, constructed via builder.
- * @param admin Administrator's name.
- * @param owner Volume owner's name
- * @param quota Volume Quota.
- * @param acls User to access rights map.
- */
- private VolumeArgs(String admin, String owner,
- String quota, List<OzoneAcl> acls) {
- this.admin = admin;
- this.owner = owner;
- this.quota = quota;
- this.acls = acls;
- }
-
- /**
- * Returns the Admin Name.
- * @return String.
- */
- public String getAdmin() {
- return admin;
- }
-
- /**
- * Returns the owner Name.
- * @return String
- */
- public String getOwner() {
- return owner;
- }
-
- /**
- * Returns Volume Quota.
- * @return Quota.
- */
- public String getQuota() {
- return quota;
- }
-
- public List<OzoneAcl> getAcls() {
- return acls;
- }
- /**
- * Returns new builder class that builds a KsmVolumeArgs.
- *
- * @return Builder
- */
- public static VolumeArgs.Builder newBuilder() {
- return new VolumeArgs.Builder();
- }
-
- /**
- * Builder for KsmVolumeArgs.
- */
- public static class Builder {
- private String adminName;
- private String ownerName;
- private String volumeQuota;
- private List<OzoneAcl> listOfAcls;
-
-
- public VolumeArgs.Builder setAdmin(String admin) {
- this.adminName = admin;
- return this;
- }
-
- public VolumeArgs.Builder setOwner(String owner) {
- this.ownerName = owner;
- return this;
- }
-
- public VolumeArgs.Builder setQuota(String quota) {
- this.volumeQuota = quota;
- return this;
- }
-
- public VolumeArgs.Builder setAcls(List<OzoneAcl> acls)
- throws IOException {
- this.listOfAcls = acls;
- return this;
- }
-
- /**
- * Constructs a CreateVolumeArgument.
- * @return CreateVolumeArgs.
- */
- public VolumeArgs build() {
- return new VolumeArgs(adminName, ownerName, volumeQuota, listOfAcls);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
deleted file mode 100644
index afe5e45..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/ozone/client/io/ChunkGroupInputStream.java
+++ /dev/null
@@ -1,319 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.client.io;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.fs.FSExceptionMessages;
-import org.apache.hadoop.fs.Seekable;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolClientSideTranslatorPB;
-import org.apache.hadoop.scm.storage.ChunkInputStream;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.ratis.util.Preconditions;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.EOFException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * Maintaining a list of ChunkInputStream. Read based on offset.
- */
-public class ChunkGroupInputStream extends InputStream implements Seekable {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(ChunkGroupInputStream.class);
-
- private static final int EOF = -1;
-
- private final ArrayList<ChunkInputStreamEntry> streamEntries;
- // streamOffset[i] stores the offset at which chunkInputStream i stores
- // data in the key
- private long[] streamOffset = null;
- private int currentStreamIndex;
- private long length = 0;
- private boolean closed = false;
- private String key;
-
- public ChunkGroupInputStream() {
- streamEntries = new ArrayList<>();
- currentStreamIndex = 0;
- }
-
- @VisibleForTesting
- public synchronized int getCurrentStreamIndex() {
- return currentStreamIndex;
- }
-
- @VisibleForTesting
- public long getRemainingOfIndex(int index) {
- return streamEntries.get(index).getRemaining();
- }
-
- /**
- * Append another stream to the end of the list.
- *
- * @param stream the stream instance.
- * @param streamLength the max number of bytes that should be written to this
- * stream.
- */
- public synchronized void addStream(ChunkInputStream stream,
- long streamLength) {
- streamEntries.add(new ChunkInputStreamEntry(stream, streamLength));
- }
-
-
- @Override
- public synchronized int read() throws IOException {
- byte[] buf = new byte[1];
- if (read(buf, 0, 1) == EOF) {
- return EOF;
- }
- return Byte.toUnsignedInt(buf[0]);
- }
-
- @Override
- public synchronized int read(byte[] b, int off, int len) throws IOException {
- checkNotClosed();
- if (b == null) {
- throw new NullPointerException();
- }
- if (off < 0 || len < 0 || len > b.length - off) {
- throw new IndexOutOfBoundsException();
- }
- if (len == 0) {
- return 0;
- }
- int totalReadLen = 0;
- while (len > 0) {
- if (streamEntries.size() <= currentStreamIndex) {
- return totalReadLen == 0 ? EOF : totalReadLen;
- }
- ChunkInputStreamEntry current = streamEntries.get(currentStreamIndex);
- int readLen = Math.min(len, (int)current.getRemaining());
- int actualLen = current.read(b, off, readLen);
- // this means the underlying stream has nothing at all, return
- if (actualLen == EOF) {
- return totalReadLen > 0 ? totalReadLen : EOF;
- }
- totalReadLen += actualLen;
- // this means there is no more data to read beyond this point, return
- if (actualLen != readLen) {
- return totalReadLen;
- }
- off += readLen;
- len -= readLen;
- if (current.getRemaining() <= 0) {
- currentStreamIndex += 1;
- }
- }
- return totalReadLen;
- }
-
- @Override
- public void seek(long pos) throws IOException {
- checkNotClosed();
- if (pos < 0 || pos >= length) {
- if (pos == 0) {
- // It is possible for length and pos to be zero in which case
- // seek should return instead of throwing exception
- return;
- }
- throw new EOFException(
- "EOF encountered at pos: " + pos + " for key: " + key);
- }
- Preconditions.assertTrue(currentStreamIndex >= 0);
- if (currentStreamIndex >= streamEntries.size()) {
- currentStreamIndex = Arrays.binarySearch(streamOffset, pos);
- } else if (pos < streamOffset[currentStreamIndex]) {
- currentStreamIndex =
- Arrays.binarySearch(streamOffset, 0, currentStreamIndex, pos);
- } else if (pos >= streamOffset[currentStreamIndex] + streamEntries
- .get(currentStreamIndex).length) {
- currentStreamIndex = Arrays
- .binarySearch(streamOffset, currentStreamIndex + 1,
- streamEntries.size(), pos);
- }
- if (currentStreamIndex < 0) {
- // Binary search returns -insertionPoint - 1 if element is not present
- // in the array. insertionPoint is the point at which element would be
- // inserted in the sorted array. We need to adjust the currentStreamIndex
- // accordingly so that currentStreamIndex = insertionPoint - 1
- currentStreamIndex = -currentStreamIndex - 2;
- }
- // seek to the proper offset in the ChunkInputStream
- streamEntries.get(currentStreamIndex)
- .seek(pos - streamOffset[currentStreamIndex]);
- }
-
- @Override
- public long getPos() throws IOException {
- return length == 0 ? 0 :
- streamOffset[currentStreamIndex] + streamEntries.get(currentStreamIndex)
- .getPos();
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
-
- @Override
- public int available() throws IOException {
- checkNotClosed();
- long remaining = length - getPos();
- return remaining <= Integer.MAX_VALUE ? (int) remaining : Integer.MAX_VALUE;
- }
-
- @Override
- public void close() throws IOException {
- closed = true;
- for (int i = 0; i < streamEntries.size(); i++) {
- streamEntries.get(i).close();
- }
- }
-
- /**
- * Encapsulates ChunkInputStream.
- */
- public static class ChunkInputStreamEntry extends InputStream
- implements Seekable {
-
- private final ChunkInputStream chunkInputStream;
- private final long length;
- private long currentPosition;
-
- public ChunkInputStreamEntry(ChunkInputStream chunkInputStream,
- long length) {
- this.chunkInputStream = chunkInputStream;
- this.length = length;
- this.currentPosition = 0;
- }
-
- synchronized long getRemaining() {
- return length - currentPosition;
- }
-
- @Override
- public synchronized int read(byte[] b, int off, int len)
- throws IOException {
- int readLen = chunkInputStream.read(b, off, len);
- currentPosition += readLen;
- return readLen;
- }
-
- @Override
- public synchronized int read() throws IOException {
- int data = chunkInputStream.read();
- currentPosition += 1;
- return data;
- }
-
- @Override
- public synchronized void close() throws IOException {
- chunkInputStream.close();
- }
-
- @Override
- public void seek(long pos) throws IOException {
- chunkInputStream.seek(pos);
- }
-
- @Override
- public long getPos() throws IOException {
- return chunkInputStream.getPos();
- }
-
- @Override
- public boolean seekToNewSource(long targetPos) throws IOException {
- return false;
- }
- }
-
- public static LengthInputStream getFromKsmKeyInfo(KsmKeyInfo keyInfo,
- XceiverClientManager xceiverClientManager,
- StorageContainerLocationProtocolClientSideTranslatorPB
- storageContainerLocationClient, String requestId)
- throws IOException {
- long length = 0;
- String containerKey;
- ChunkGroupInputStream groupInputStream = new ChunkGroupInputStream();
- groupInputStream.key = keyInfo.getKeyName();
- List<KsmKeyLocationInfo> keyLocationInfos =
- keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
- groupInputStream.streamOffset = new long[keyLocationInfos.size()];
- for (int i = 0; i < keyLocationInfos.size(); i++) {
- KsmKeyLocationInfo ksmKeyLocationInfo = keyLocationInfos.get(i);
- String containerName = ksmKeyLocationInfo.getContainerName();
- Pipeline pipeline =
- storageContainerLocationClient.getContainer(containerName);
- XceiverClientSpi xceiverClient =
- xceiverClientManager.acquireClient(pipeline);
- boolean success = false;
- containerKey = ksmKeyLocationInfo.getBlockID();
- try {
- LOG.debug("get key accessing {} {}",
- xceiverClient.getPipeline().getContainerName(), containerKey);
- groupInputStream.streamOffset[i] = length;
- ContainerProtos.KeyData containerKeyData = OzoneContainerTranslation
- .containerKeyDataForRead(
- xceiverClient.getPipeline().getContainerName(), containerKey);
- ContainerProtos.GetKeyResponseProto response = ContainerProtocolCalls
- .getKey(xceiverClient, containerKeyData, requestId);
- List<ContainerProtos.ChunkInfo> chunks =
- response.getKeyData().getChunksList();
- for (ContainerProtos.ChunkInfo chunk : chunks) {
- length += chunk.getLen();
- }
- success = true;
- ChunkInputStream inputStream = new ChunkInputStream(
- containerKey, xceiverClientManager, xceiverClient,
- chunks, requestId);
- groupInputStream.addStream(inputStream,
- ksmKeyLocationInfo.getLength());
- } finally {
- if (!success) {
- xceiverClientManager.releaseClient(xceiverClient);
- }
- }
- }
- groupInputStream.length = length;
- return new LengthInputStream(groupInputStream, length);
- }
-
- /**
- * Verify that the input stream is open. Non blocking; this gives
- * the last state of the volatile {@link #closed} field.
- * @throws IOException if the connection is closed.
- */
- private void checkNotClosed() throws IOException {
- if (closed) {
- throw new IOException(
- ": " + FSExceptionMessages.STREAM_IS_CLOSED + " Key: " + key);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
new file mode 100644
index 0000000..26c174f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/DiskBlock.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import java.nio.ByteBuffer;
+
+/**
+ * Impl class for LogicalBlock.
+ */
+public class DiskBlock implements LogicalBlock {
+ private ByteBuffer data;
+ private long blockID;
+ private boolean persisted;
+
+ /**
+ * Constructs a DiskBlock Class from the following params.
+ * @param blockID - 64-bit block ID
+ * @param data - Byte Array
+ * @param persisted - Flag which tells us if this is persisted to remote
+ */
+ public DiskBlock(long blockID, byte[] data, boolean persisted) {
+ if (data !=null) {
+ this.data = ByteBuffer.wrap(data);
+ }
+ this.blockID = blockID;
+ this.persisted = persisted;
+ }
+
+ @Override
+ public ByteBuffer getData() {
+ return data;
+ }
+
+ /**
+ * Frees the byte buffer since we don't need it any more.
+ */
+ @Override
+ public void clearData() {
+ data.clear();
+ }
+
+ @Override
+ public long getBlockID() {
+ return blockID;
+ }
+
+ @Override
+ public boolean isPersisted() {
+ return persisted;
+ }
+
+ /**
+ * Sets the value of persisted.
+ * @param value - True if this has been persisted to container, false
+ * otherwise.
+ */
+ public void setPersisted(boolean value) {
+ persisted = value;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
new file mode 100644
index 0000000..7d6e172
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/SyncBlockReader.java
@@ -0,0 +1,263 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdsl.protocol.proto.ContainerProtos;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.util.concurrent.HadoopThreadPoolExecutor;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.iq80.leveldb.DBException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Reads blocks from the container via the local cache.
+ */
+public class SyncBlockReader {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SyncBlockReader.class);
+
+ /**
+ * Update Queue - The reason why we have the queue is that we want to
+ * return the block as soon as we read it from the containers. This queue
+ * is work queue which will take the read block and update the cache.
+ * During testing we found levelDB is slow during writes, hence we wanted
+ * to return as block as soon as possible and update levelDB asynchronously.
+ */
+ private final static int QUEUE_SIZE = 1024;
+ /**
+ * Config.
+ */
+ private final Configuration conf;
+ /**
+ * The parent cache this reader is operating against.
+ */
+ private final CBlockLocalCache parentCache;
+ private final BlockingQueue<Runnable> updateQueue;
+
+ /**
+ * executor is used for running LevelDB updates. In future, we might do
+ * read-aheads and this pool is useful for that too. The reason why we
+ * don't share an executor for reads and writes is because the write task
+ * is couple of magnitude slower than read task. So we don't want the
+ * update DB to queue up behind the writes.
+ */
+ private final ThreadPoolExecutor executor;
+
+ /**
+ * Number of threads that pool starts with.
+ */
+ private final int corePoolSize = 1;
+ /**
+ * Maximum number of threads our pool will ever create.
+ */
+ private final int maxPoolSize = 10;
+ /**
+ * The idle time a thread hangs around waiting for work. if we don't find
+ * new work in 60 seconds the worker thread is killed.
+ */
+ private final long keepAlive = 60L;
+
+ /**
+ * Constructs a SyncBlock reader.
+ *
+ * @param conf - Configuration
+ * @param cache - Cache
+ */
+ public SyncBlockReader(Configuration conf, CBlockLocalCache cache) {
+ this.conf = conf;
+ this.parentCache = cache;
+ updateQueue = new ArrayBlockingQueue<>(QUEUE_SIZE, true);
+ ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("SyncBlockReader Thread #%d").setDaemon(true).build();
+ executor = new HadoopThreadPoolExecutor(
+ corePoolSize, maxPoolSize, keepAlive, TimeUnit.SECONDS,
+ updateQueue, workerThreadFactory,
+ new ThreadPoolExecutor.CallerRunsPolicy());
+ }
+
+ /**
+ * Returns the cache DB.
+ *
+ * @return LevelDB
+ */
+ private LevelDBStore getCacheDB() {
+ return parentCache.getCacheDB();
+ }
+
+ /**
+ * Returns data from the local cache if found, else reads from the remote
+ * container.
+ *
+ * @param blockID - blockID
+ * @return LogicalBlock
+ */
+ LogicalBlock readBlock(long blockID) throws IOException {
+ XceiverClientSpi client = null;
+ byte[] data = getblockFromDB(blockID);
+ if (data != null) {
+ parentCache.getTargetMetrics().incNumReadCacheHits();
+ return new DiskBlock(blockID, data, false);
+ }
+
+ parentCache.getTargetMetrics().incNumReadCacheMiss();
+ try {
+ client = parentCache.getClientManager()
+ .acquireClient(parentCache.getPipeline(blockID));
+ LogicalBlock block = getBlockFromContainer(blockID, client);
+ return block;
+ } catch (Exception ex) {
+ parentCache.getTargetMetrics().incNumFailedReadBlocks();
+ LOG.error("read failed for BlockId: {}", blockID, ex);
+ throw ex;
+ } finally {
+ if (client != null) {
+ parentCache.getClientManager().releaseClient(client);
+ }
+ }
+ }
+
+ /**
+ * Gets data from the DB if it exists.
+ *
+ * @param blockID - block id
+ * @return data
+ */
+ private byte[] getblockFromDB(long blockID) {
+ try {
+ if(parentCache.isShortCircuitIOEnabled()) {
+ long startTime = Time.monotonicNow();
+ byte[] data = getCacheDB().get(Longs.toByteArray(blockID));
+ long endTime = Time.monotonicNow();
+
+ if (parentCache.isTraceEnabled()) {
+ parentCache.getTracer().info(
+ "Task=ReadTaskDBRead,BlockID={},SHA={},Time={}",
+ blockID, (data != null && data.length > 0)
+ ? DigestUtils.sha256Hex(data) : null,
+ endTime - startTime);
+ }
+ parentCache.getTargetMetrics().updateDBReadLatency(
+ endTime - startTime);
+ return data;
+ }
+
+
+ } catch (DBException dbe) {
+ LOG.error("Error while reading from cacheDB.", dbe);
+ throw dbe;
+ }
+ return null;
+ }
+
+
+ /**
+ * Returns a block from a Remote Container. if the key is not found on a
+ * remote container we just return a block initialzied with zeros.
+ *
+ * @param blockID - blockID
+ * @param client - client
+ * @return LogicalBlock
+ * @throws IOException
+ */
+ private LogicalBlock getBlockFromContainer(long blockID,
+ XceiverClientSpi client) throws IOException {
+ String containerName = parentCache.getPipeline(blockID).getContainerName();
+ try {
+ long startTime = Time.monotonicNow();
+ ContainerProtos.GetSmallFileResponseProto response =
+ ContainerProtocolCalls.readSmallFile(client, containerName,
+ Long.toString(blockID), parentCache.getTraceID(blockID));
+ long endTime = Time.monotonicNow();
+ if (parentCache.isTraceEnabled()) {
+ parentCache.getTracer().info(
+ "Task=ReadTaskContainerRead,BlockID={},SHA={},Time={}",
+ blockID, response.getData().getData().toByteArray().length > 0 ?
+ DigestUtils.sha256Hex(response.getData()
+ .getData().toByteArray()) : null, endTime - startTime);
+ }
+
+ parentCache.getTargetMetrics().updateContainerReadLatency(
+ endTime - startTime);
+ DiskBlock block = new DiskBlock(blockID,
+ response.getData().getData().toByteArray(), false);
+
+ if(parentCache.isShortCircuitIOEnabled()) {
+ queueUpdateTask(block);
+ }
+
+ return block;
+ } catch (IOException ex) {
+ if (ex instanceof StorageContainerException) {
+ parentCache.getTargetMetrics().incNumReadLostBlocks();
+ StorageContainerException sce = (StorageContainerException) ex;
+ if (sce.getResult() == ContainerProtos.Result.NO_SUCH_KEY ||
+ sce.getResult() == ContainerProtos.Result.IO_EXCEPTION) {
+ return new DiskBlock(blockID, new byte[parentCache.getBlockSize()],
+ false);
+ }
+ }
+ throw ex;
+ }
+ }
+
+ /**
+ * Updates the cache with the block that we just read.
+ *
+ * @param block
+ */
+ private void queueUpdateTask(final DiskBlock block) {
+ Runnable updateTask = () -> {
+ if(block.getData().array().length > 0) {
+ getCacheDB().put(Longs.toByteArray(block.getBlockID()),
+ block.getData().array());
+ block.setPersisted(true);
+ } else {
+ LOG.error("Refusing to update the a null block in the local cache.");
+ }
+ };
+ if (this.executor.isShutdown() || this.executor.isTerminated()) {
+ LOG.error("Thread executor is not running.");
+ } else {
+ this.executor.submit(updateTask);
+ }
+ }
+
+ /**
+ * This is a read operation, we don't care if we updated the cache with the
+ * last block e read.
+ */
+ void shutdown() {
+ this.executor.shutdownNow();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
new file mode 100644
index 0000000..dfac110
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
new file mode 100644
index 0000000..47f76b8
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
new file mode 100644
index 0000000..85f8d6f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
new file mode 100644
index 0000000..72b215b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.kubernetes;
+
+import com.google.gson.reflect.TypeToken;
+import com.squareup.okhttp.RequestBody;
+import io.kubernetes.client.ApiClient;
+import io.kubernetes.client.ApiException;
+import io.kubernetes.client.Configuration;
+import io.kubernetes.client.apis.CoreV1Api;
+import io.kubernetes.client.models.V1ISCSIVolumeSource;
+import io.kubernetes.client.models.V1ObjectMeta;
+import io.kubernetes.client.models.V1ObjectReference;
+import io.kubernetes.client.models.V1PersistentVolume;
+import io.kubernetes.client.models.V1PersistentVolumeClaim;
+import io.kubernetes.client.models.V1PersistentVolumeSpec;
+import io.kubernetes.client.util.Config;
+import io.kubernetes.client.util.Watch;
+import okio.Buffer;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.exception.CBlockException;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.storage.StorageManager;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.SocketTimeoutException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_KUBERNETES_CBLOCK_USER;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY;
+
+/**
+ * Kubernetes Dynamic Persistent Volume provisioner.
+ *
+ * Listens on the kubernetes feed and creates the appropriate cblock AND
+ * kubernetes PersistentVolume according to the created PersistentVolumeClaims.
+ */
+public class DynamicProvisioner implements Runnable{
+
+ protected static final Logger LOGGER =
+ LoggerFactory.getLogger(DynamicProvisioner.class);
+
+ private static final String STORAGE_CLASS = "cblock";
+
+ private static final String PROVISIONER_ID = "hadoop.apache.org/cblock";
+ private static final String KUBERNETES_PROVISIONER_KEY =
+ "volume.beta.kubernetes.io/storage-provisioner";
+ private static final String KUBERNETES_BIND_COMPLETED_KEY =
+ "pv.kubernetes.io/bind-completed";
+
+ private boolean running = true;
+
+ private final StorageManager storageManager;
+
+ private String kubernetesConfigFile;
+
+ private String externalIp;
+
+ private int externalPort;
+
+ private String cblockUser;
+
+ private CoreV1Api api;
+
+ private ApiClient client;
+
+ private Thread watcherThread;
+
+ public DynamicProvisioner(OzoneConfiguration ozoneConf,
+ StorageManager storageManager) throws IOException {
+ this.storageManager = storageManager;
+
+ kubernetesConfigFile = ozoneConf
+ .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY);
+
+ String jscsiServerAddress = ozoneConf
+ .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
+ DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
+
+ externalIp = ozoneConf.
+ getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
+
+ externalPort = ozoneConf.
+ getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
+ DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
+
+ cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER,
+ DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT);
+
+
+ }
+
+ public void init() throws IOException {
+ if (kubernetesConfigFile != null) {
+ client = Config.fromConfig(kubernetesConfigFile);
+ } else {
+ client = Config.fromCluster();
+ }
+ client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS);
+ Configuration.setDefaultApiClient(client);
+ api = new CoreV1Api();
+
+ watcherThread = new Thread(this);
+ watcherThread.setName("DynamicProvisioner");
+ watcherThread.setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ LOGGER.info("Starting kubernetes dynamic provisioner.");
+ while (running) {
+ String resourceVersion = null;
+ try {
+
+ Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client,
+ api.listPersistentVolumeClaimForAllNamespacesCall(null,
+ null,
+ false,
+ null,
+ null,
+ null,
+ resourceVersion,
+ null,
+ true,
+ null,
+ null),
+ new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() {
+ }.getType());
+
+
+ //check the new pvc resources, and create cblock + pv if needed
+ for (Watch.Response<V1PersistentVolumeClaim> item : watch) {
+ V1PersistentVolumeClaim claim = item.object;
+
+ if (isPvMissingForPvc(claim)) {
+
+ LOGGER.info("Provisioning volumes for PVC {}/{}",
+ claim.getMetadata().getNamespace(),
+ claim.getMetadata().getName());
+
+ if (LOGGER.isDebugEnabled()) {
+ RequestBody request =
+ api.getApiClient().serialize(claim, "application/json");
+
+ final Buffer buffer = new Buffer();
+ request.writeTo(buffer);
+ LOGGER.debug("New PVC is detected: " + buffer.readUtf8());
+ }
+
+ String volumeName = createVolumeName(claim);
+
+ long size = CblockUtils.parseSize(
+ claim.getSpec().getResources().getRequests().get("storage"));
+
+ createCBlock(volumeName, size);
+ createPersistentVolumeFromPVC(item.object, volumeName);
+ }
+ }
+ } catch (Exception ex) {
+ if (ex.getCause() != null && ex
+ .getCause() instanceof SocketTimeoutException) {
+ //This is normal. We are connection to the kubernetes server and the
+ //connection should be reopened time to time...
+ LOGGER.debug("Time exception occured", ex);
+ } else {
+ LOGGER.error("Error on provisioning persistent volumes.", ex);
+ try {
+ //we can try again in the main loop
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ LOGGER.error("Error on sleeping after an error.", e);
+ }
+ }
+ }
+ }
+ }
+
+ private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) {
+
+ Map<String, String> annotations = claim.getMetadata().getAnnotations();
+
+ return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS
+ .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID
+ .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes"
+ .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY));
+ }
+
+ @VisibleForTesting
+ protected String createVolumeName(V1PersistentVolumeClaim claim) {
+ return claim.getMetadata().getName() + "-" + claim.getMetadata()
+ .getUid();
+ }
+
+ public void stop() {
+ running = false;
+ try {
+ watcherThread.join(60000);
+ } catch (InterruptedException e) {
+ LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e);
+ }
+ }
+
+ private void createCBlock(String volumeName, long size)
+ throws CBlockException {
+
+ MountVolumeResponse mountVolumeResponse =
+ storageManager.isVolumeValid(cblockUser, volumeName);
+ if (!mountVolumeResponse.getIsValid()) {
+ storageManager
+ .createVolume(cblockUser, volumeName, size, 4 * 1024);
+ }
+ }
+
+ private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim,
+ String volumeName) throws ApiException, IOException {
+
+ V1PersistentVolume v1PersistentVolume =
+ persitenceVolumeBuilder(claim, volumeName);
+
+ if (LOGGER.isDebugEnabled()) {
+ RequestBody request =
+ api.getApiClient().serialize(v1PersistentVolume, "application/json");
+
+ final Buffer buffer = new Buffer();
+ request.writeTo(buffer);
+ LOGGER.debug("Creating new PV: " + buffer.readUtf8());
+ }
+ api.createPersistentVolume(v1PersistentVolume, null);
+ }
+
+ protected V1PersistentVolume persitenceVolumeBuilder(
+ V1PersistentVolumeClaim claim,
+ String volumeName) {
+
+ V1PersistentVolume v1PersistentVolume = new V1PersistentVolume();
+ v1PersistentVolume.setKind("PersistentVolume");
+ v1PersistentVolume.setApiVersion("v1");
+
+ V1ObjectMeta metadata = new V1ObjectMeta();
+ metadata.setName(volumeName);
+ metadata.setNamespace(claim.getMetadata().getNamespace());
+ metadata.setAnnotations(new HashMap<>());
+
+ metadata.getAnnotations()
+ .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID);
+
+ metadata.getAnnotations()
+ .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS);
+
+ v1PersistentVolume.setMetadata(metadata);
+
+ V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec();
+
+ spec.setCapacity(new HashMap<>());
+ spec.getCapacity().put("storage",
+ claim.getSpec().getResources().getRequests().get("storage"));
+
+ spec.setAccessModes(new ArrayList<>());
+ spec.getAccessModes().add("ReadWriteOnce");
+
+ V1ObjectReference claimRef = new V1ObjectReference();
+ claimRef.setName(claim.getMetadata().getName());
+ claimRef.setNamespace(claim.getMetadata().getNamespace());
+ claimRef.setKind(claim.getKind());
+ claimRef.setApiVersion(claim.getApiVersion());
+ claimRef.setUid(claim.getMetadata().getUid());
+ spec.setClaimRef(claimRef);
+
+ spec.persistentVolumeReclaimPolicy("Delete");
+
+ V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource();
+ iscsi.setIqn(cblockUser + ":" + volumeName);
+ iscsi.setLun(0);
+ iscsi.setFsType("ext4");
+ String portal = externalIp + ":" + externalPort;
+ iscsi.setTargetPortal(portal);
+ iscsi.setPortals(new ArrayList<>());
+ iscsi.getPortals().add(portal);
+
+ spec.iscsi(iscsi);
+ v1PersistentVolume.setSpec(spec);
+ return v1PersistentVolume;
+ }
+
+
+ @VisibleForTesting
+ protected CoreV1Api getApi() {
+ return api;
+ }
+
+ public void start() {
+ watcherThread.start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
new file mode 100644
index 0000000..3ec5aab
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains helper classes to run hadoop cluster in kubernetes
+ * environment.
+ */
+package org.apache.hadoop.cblock.kubernetes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
new file mode 100644
index 0000000..00064a6
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+/**
+ *
+ * The internal representation of a container maintained by CBlock server.
+ * Include enough information to exactly identify a container for read/write
+ * operation.
+ *
+ * NOTE that this class is work-in-progress. Depends on HDFS-7240 container
+ * implementation. Currently only to allow testing.
+ */
+public class ContainerDescriptor {
+ private final String containerID;
+ // the index of this container with in a volume
+ // on creation, there may be no way to know the index of the container
+ // as it is a volume specific information
+ private int containerIndex;
+ private Pipeline pipeline;
+
+ public ContainerDescriptor(String containerID) {
+ this.containerID = containerID;
+ }
+
+ public ContainerDescriptor(String containerID, int containerIndex) {
+ this.containerID = containerID;
+ this.containerIndex = containerIndex;
+ }
+
+ public void setContainerIndex(int idx) {
+ this.containerIndex = idx;
+ }
+
+ public String getContainerID() {
+ return containerID;
+ }
+
+ public void setPipeline(Pipeline pipeline) {
+ this.pipeline = pipeline;
+ }
+
+ public Pipeline getPipeline() {
+ return pipeline;
+ }
+
+ public int getContainerIndex() {
+ return containerIndex;
+ }
+
+ public long getUtilization() {
+ return 0;
+ }
+
+ public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() {
+ CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder =
+ CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
+ builder.setContainerID(containerID);
+ builder.setIndex(containerIndex);
+ if (pipeline != null) {
+ builder.setPipeline(pipeline.getProtobufMessage());
+ }
+ return builder.build();
+ }
+
+ public static ContainerDescriptor fromProtobuf(byte[] data)
+ throws InvalidProtocolBufferException {
+ CBlockClientServerProtocolProtos.ContainerIDProto id =
+ CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data);
+ return new ContainerDescriptor(id.getContainerID(),
+ (int)id.getIndex());
+ }
+
+ @Override
+ public int hashCode() {
+ return containerID.hashCode()*37 + containerIndex;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o != null && o instanceof ContainerDescriptor) {
+ ContainerDescriptor other = (ContainerDescriptor)o;
+ return containerID.equals(other.containerID) &&
+ containerIndex == other.containerIndex;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
new file mode 100644
index 0000000..4f5930d
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * The internal representation maintained by CBlock server as the info for
+ * a volume. Contains the list of containers belonging to this volume.
+ *
+ * Many methods of this class is made such that the volume information (
+ * including container list) can be easily transformed into a Json string
+ * that can be stored/parsed from a persistent store for cblock server
+ * persistence.
+ *
+ * This class is still work-in-progress.
+ */
+public class VolumeDescriptor {
+ // The main data structure is the container location map
+ // other thing are mainly just information
+
+ // since only one operation at a time is allowed, no
+ // need to consider concurrency control here
+
+ // key is container id
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(VolumeDescriptor.class);
+
+ private ConcurrentHashMap<String, ContainerDescriptor> containerMap;
+ private String userName;
+ private int blockSize;
+ private long volumeSize;
+ private String volumeName;
+ // this is essentially the ordered keys of containerMap
+ // which is kind of redundant information. But since we
+ // are likely to access it frequently based on ordering.
+ // keeping this copy to avoid having to sort the key every
+ // time
+ private List<String> containerIdOrdered;
+
+ /**
+ * This is not being called explicitly, but this is necessary as
+ * it will be called by the parse method implicitly when
+ * reconstructing the object from json string. The get*() methods
+ * and set*() methods are for the same purpose also.
+ */
+ public VolumeDescriptor() {
+ this(null, null, 0, 0);
+ }
+
+ public VolumeDescriptor(String userName, String volumeName, long volumeSize,
+ int blockSize) {
+ this.containerMap = new ConcurrentHashMap<>();
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.blockSize = blockSize;
+ this.volumeSize = volumeSize;
+ this.containerIdOrdered = new LinkedList<>();
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public void setUserName(String userName) {
+ this.userName = userName;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public void setVolumeName(String volumeName) {
+ this.volumeName = volumeName;
+ }
+
+ public long getVolumeSize() {
+ return volumeSize;
+ }
+
+ public void setVolumeSize(long volumeSize) {
+ this.volumeSize = volumeSize;
+ }
+
+ public int getBlockSize() {
+ return blockSize;
+ }
+
+ public void setBlockSize(int blockSize) {
+ this.blockSize = blockSize;
+ }
+
+ public void setContainerIDs(ArrayList<String> containerIDs) {
+ containerIdOrdered.addAll(containerIDs);
+ }
+
+ public void addContainer(ContainerDescriptor containerDescriptor) {
+ containerMap.put(containerDescriptor.getContainerID(),
+ containerDescriptor);
+ }
+
+
+ public HashMap<String, Pipeline> getPipelines() {
+ HashMap<String, Pipeline> pipelines = new HashMap<>();
+ for (Map.Entry<String, ContainerDescriptor> entry :
+ containerMap.entrySet()) {
+ pipelines.put(entry.getKey(), entry.getValue().getPipeline());
+ }
+ return pipelines;
+ }
+
+ public boolean isEmpty() {
+ VolumeInfo info = getInfo();
+ return info.getUsage() == 0;
+ }
+
+ public VolumeInfo getInfo() {
+ // TODO : need to actually go through all containers of this volume and
+ // ask for their utilization.
+ long utilization = 0;
+ for (Map.Entry<String, ContainerDescriptor> entry :
+ containerMap.entrySet()) {
+ utilization += entry.getValue().getUtilization();
+ }
+ return new VolumeInfo(this.userName, this.volumeName,
+ this.volumeSize, this.blockSize,
+ utilization * blockSize);
+ }
+
+ public String[] getContainerIDs() {
+ //ArrayList<Long> ids = new ArrayList(containerMap.keySet());
+ //return ids.toArray(new Long[ids.size()]);
+ return containerIdOrdered.toArray(new String[containerIdOrdered.size()]);
+ }
+
+ public List<String> getContainerIDsList() {
+ return new ArrayList<>(containerIdOrdered);
+ }
+
+ public List<Pipeline> getContainerPipelines() {
+ Map<String, Pipeline> tmp = getPipelines();
+ List<Pipeline> pipelineList = new LinkedList<>();
+ for (String containerIDString : containerIdOrdered) {
+ pipelineList.add(tmp.get(containerIDString));
+ }
+ return pipelineList;
+ }
+
+ @Override
+ public String toString() {
+ String string = "";
+ string += "Username:" + userName + "\n";
+ string += "VolumeName:" + volumeName + "\n";
+ string += "VolumeSize:" + volumeSize + "\n";
+ string += "blockSize:" + blockSize + "\n";
+ string += "containerIds:" + containerIdOrdered + "\n";
+ string += "containerIdsWithObject:" + containerMap.keySet();
+ return string;
+ }
+
+ public CBlockClientServerProtocolProtos.MountVolumeResponseProto
+ toProtobuf() {
+ CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume =
+ CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder();
+ volume.setIsValid(true);
+ volume.setVolumeName(volumeName);
+ volume.setUserName(userName);
+ volume.setVolumeSize(volumeSize);
+ volume.setBlockSize(blockSize);
+ for (String containerIDString : containerIdOrdered) {
+ ContainerDescriptor containerDescriptor = containerMap.get(
+ containerIDString);
+ volume.addAllContainerIDs(containerDescriptor.toProtobuf());
+ }
+ return volume.build();
+ }
+
+ public static VolumeDescriptor fromProtobuf(byte[] data)
+ throws InvalidProtocolBufferException {
+ CBlockClientServerProtocolProtos.MountVolumeResponseProto volume =
+ CBlockClientServerProtocolProtos.MountVolumeResponseProto
+ .parseFrom(data);
+ String userName = volume.getUserName();
+ String volumeName = volume.getVolumeName();
+ long volumeSize = volume.getVolumeSize();
+ int blockSize = volume.getBlockSize();
+ VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName,
+ volumeName, volumeSize, blockSize);
+ List<CBlockClientServerProtocolProtos.ContainerIDProto> containers
+ = volume.getAllContainerIDsList();
+
+ String[] containerOrdering = new String[containers.size()];
+
+ for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto :
+ containers) {
+ ContainerDescriptor containerDescriptor = new ContainerDescriptor(
+ containerProto.getContainerID(),
+ (int)containerProto.getIndex());
+ if(containerProto.hasPipeline()) {
+ containerDescriptor.setPipeline(
+ Pipeline.getFromProtoBuf(containerProto.getPipeline()));
+ }
+ volumeDescriptor.addContainer(containerDescriptor);
+ containerOrdering[containerDescriptor.getContainerIndex()] =
+ containerDescriptor.getContainerID();
+ }
+ volumeDescriptor.setContainerIDs(
+ new ArrayList<>(Arrays.asList(containerOrdering)));
+ return volumeDescriptor;
+ }
+
+ @Override
+ public int hashCode() {
+ return userName.hashCode()*37 + volumeName.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o != null && o instanceof VolumeDescriptor) {
+ VolumeDescriptor other = (VolumeDescriptor)o;
+ if (!userName.equals(other.getUserName()) ||
+ !volumeName.equals(other.getVolumeName()) ||
+ volumeSize != other.getVolumeSize() ||
+ blockSize != other.getBlockSize()) {
+ return false;
+ }
+ if (containerIdOrdered.size() != other.containerIdOrdered.size() ||
+ containerMap.size() != other.containerMap.size()) {
+ return false;
+ }
+ for (int i = 0; i<containerIdOrdered.size(); i++) {
+ if (!containerIdOrdered.get(i).equals(
+ other.containerIdOrdered.get(i))) {
+ return false;
+ }
+ }
+ return containerMap.equals(other.containerMap);
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
new file mode 100644
index 0000000..7f50c41
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
+
+/**
+ * A wrapper class that represents the information about a volume. Used in
+ * communication between CBlock client and CBlock server only.
+ */
+public class VolumeInfo {
+ private final String userName;
+ private final String volumeName;
+ private final long volumeSize;
+ private final long blockSize;
+ private final long usage;
+
+ public VolumeInfo(String userName, String volumeName, long volumeSize,
+ long blockSize, long usage) {
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.volumeSize = volumeSize;
+ this.blockSize = blockSize;
+ this.usage = usage;
+ }
+
+ // When listing volume, the usage will not be set.
+ public VolumeInfo(String userName, String volumeName, long volumeSize,
+ long blockSize) {
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.volumeSize = volumeSize;
+ this.blockSize = blockSize;
+ this.usage = -1;
+ }
+
+ public long getVolumeSize() {
+ return volumeSize;
+ }
+
+ public long getBlockSize() {
+ return blockSize;
+ }
+
+ public long getUsage() {
+ return usage;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ @Override
+ public String toString() {
+ return " userName:" + userName +
+ " volumeName:" + volumeName +
+ " volumeSize:" + volumeSize +
+ " blockSize:" + blockSize +
+ " (sizeInBlocks:" + volumeSize/blockSize + ")" +
+ " usageInBlocks:" + usage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
new file mode 100644
index 0000000..a331d7a
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.meta;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
new file mode 100644
index 0000000..a7d5d8b
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
new file mode 100644
index 0000000..fc40cef
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * The protocol that CBlock client side uses to talk to server side. CBlock
+ * client is the point where a volume is mounted. All the actual volume IO
+ * operations will go through CBlock client after the volume is mounted.
+ *
+ * When users mount a volume on CBlock client, CBlock client side uses this
+ * protocol to send mount request to CBlock server.
+ */
+public interface CBlockClientProtocol {
+ MountVolumeResponse mountVolume(String userName, String volumeName)
+ throws IOException;
+
+ List<VolumeInfo> listVolumes() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
new file mode 100644
index 0000000..bf00bc0
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.io.IOException;
+import java.util.List;
+
+/**
+ * CBlock uses a separate command line tool to send volume management
+ * operations to CBlock server, including create/delete/info/list volumes. This
+ * is the protocol used by the command line tool to send these requests and get
+ * responses from CBlock server.
+ */
+@InterfaceAudience.Private
+public interface CBlockServiceProtocol {
+
+ void createVolume(String userName, String volumeName,
+ long volumeSize, int blockSize) throws IOException;
+
+ void deleteVolume(String userName, String volumeName,
+ boolean force) throws IOException;
+
+ VolumeInfo infoVolume(String userName,
+ String volumeName) throws IOException;
+
+ List<VolumeInfo> listVolume(String userName) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
new file mode 100644
index 0000000..70ccd90
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
+
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * The response message of mounting a volume. Including enough information
+ * for the client to communicate (perform IO) with the volume containers
+ * directly.
+ */
+public class MountVolumeResponse {
+ private final boolean isValid;
+ private final String userName;
+ private final String volumeName;
+ private final long volumeSize;
+ private final int blockSize;
+ private List<Pipeline> containerList;
+ private HashMap<String, Pipeline> pipelineMap;
+
+ public MountVolumeResponse(boolean isValid, String userName,
+ String volumeName, long volumeSize, int blockSize,
+ List<Pipeline> containerList,
+ HashMap<String, Pipeline> pipelineMap) {
+ this.isValid = isValid;
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.volumeSize = volumeSize;
+ this.blockSize = blockSize;
+ this.containerList = containerList;
+ this.pipelineMap = pipelineMap;
+ }
+
+ public boolean getIsValid() {
+ return isValid;
+ }
+
+ public String getUserName() {
+ return userName;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public long getVolumeSize() {
+ return volumeSize;
+ }
+
+ public int getBlockSize() {
+ return blockSize;
+ }
+
+ public List<Pipeline> getContainerList() {
+ return containerList;
+ }
+
+ public HashMap<String, Pipeline> getPipelineMap() {
+ return pipelineMap;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
new file mode 100644
index 0000000..33438ec
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.proto;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
new file mode 100644
index 0000000..99f3110
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+
+/**
+ * This is the protocol CBlock client uses to talk to CBlock server.
+ * CBlock client is the mounting point of a volume. When a user mounts a
+ * volume, the cBlock client running on the local node will use this protocol
+ * to talk to CBlock server to mount the volume.
+ */
+@ProtocolInfo(protocolName =
+ "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol",
+ protocolVersion = 1)
+@InterfaceAudience.Private
+public interface CBlockClientServerProtocolPB extends
+ CBlockClientServerProtocolProtos
+ .CBlockClientServerProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..bfe2130
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The server side implementation of cblock client to server protocol.
+ */
+@InterfaceAudience.Private
+public class CBlockClientServerProtocolServerSideTranslatorPB implements
+ CBlockClientServerProtocolPB {
+
+ private final CBlockClientProtocol impl;
+
+ public CBlockClientServerProtocolServerSideTranslatorPB(
+ CBlockClientProtocol impl) {
+ this.impl = impl;
+ }
+
+ @Override
+ public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume(
+ RpcController controller,
+ CBlockClientServerProtocolProtos.MountVolumeRequestProto request)
+ throws ServiceException {
+ String userName = request.getUserName();
+ String volumeName = request.getVolumeName();
+ CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder
+ resp =
+ CBlockClientServerProtocolProtos
+ .MountVolumeResponseProto.newBuilder();
+ try {
+ MountVolumeResponse result = impl.mountVolume(userName, volumeName);
+ boolean isValid = result.getIsValid();
+ resp.setIsValid(isValid);
+ if (isValid) {
+ resp.setUserName(result.getUserName());
+ resp.setVolumeName(result.getVolumeName());
+ resp.setVolumeSize(result.getVolumeSize());
+ resp.setBlockSize(result.getBlockSize());
+ List<Pipeline> containers = result.getContainerList();
+ HashMap<String, Pipeline> pipelineMap = result.getPipelineMap();
+
+ for (int i=0; i<containers.size(); i++) {
+ CBlockClientServerProtocolProtos.ContainerIDProto.Builder id =
+ CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
+ String containerName = containers.get(i).getContainerName();
+ id.setContainerID(containerName);
+ id.setIndex(i);
+ if (pipelineMap.containsKey(containerName)) {
+ id.setPipeline(pipelineMap.get(containerName).getProtobufMessage());
+ }
+ resp.addAllContainerIDs(id.build());
+ }
+ }
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return resp.build();
+ }
+
+ @Override
+ public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes(
+ RpcController controller,
+ CBlockClientServerProtocolProtos.ListVolumesRequestProto request)
+ throws ServiceException {
+ try {
+ CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp =
+ CBlockClientServerProtocolProtos.ListVolumesResponseProto
+ .newBuilder();
+ List<VolumeInfo> volumeInfos = impl.listVolumes();
+ List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos =
+ volumeInfos.stream().map(
+ volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto
+ .newBuilder().setUserName(volumeInfo.getUserName())
+ .setBlockSize(volumeInfo.getBlockSize())
+ .setVolumeName(volumeInfo.getVolumeName())
+ .setVolumeSize(volumeInfo.getVolumeSize())
+ .setUsage(volumeInfo.getUsage()).build())
+ .collect(Collectors.toList());
+ resp.addAllVolumeEntry(convertedInfos);
+ return resp.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
new file mode 100644
index 0000000..282d6cd
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.ipc.ProtocolInfo;
+
+/**
+ * Users use a independent command line tool to talk to CBlock server for
+ * volume operations (create/delete/info/list). This is the protocol used by
+ * the the command line tool to send these requests to CBlock server.
+ */
+@ProtocolInfo(protocolName =
+ "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol",
+ protocolVersion = 1)
+@InterfaceAudience.Private
+public interface CBlockServiceProtocolPB extends
+ CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
new file mode 100644
index 0000000..8924a0c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.cblock.meta.VolumeInfo;
+import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
+import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT;
+
+/**
+ * Server side implementation of the protobuf service.
+ */
+@InterfaceAudience.Private
+public class CBlockServiceProtocolServerSideTranslatorPB
+ implements CBlockServiceProtocolPB {
+
+ private final CBlockServiceProtocol impl;
+ private static final Logger LOG =
+ LoggerFactory.getLogger(
+ CBlockServiceProtocolServerSideTranslatorPB.class);
+
+ @Override
+ public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume(
+ RpcController controller,
+ CBlockServiceProtocolProtos.CreateVolumeRequestProto request)
+ throws ServiceException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("createVolume called! volume size: " + request.getVolumeSize()
+ + " block size: " + request.getBlockSize());
+ }
+ try {
+ if (request.hasBlockSize()) {
+ impl.createVolume(request.getUserName(), request.getVolumeName(),
+ request.getVolumeSize(), request.getBlockSize());
+ } else{
+ impl.createVolume(request.getUserName(), request.getVolumeName(),
+ request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT);
+ }
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return CBlockServiceProtocolProtos.CreateVolumeResponseProto
+ .newBuilder().build();
+ }
+
+ @Override
+ public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume(
+ RpcController controller,
+ CBlockServiceProtocolProtos.DeleteVolumeRequestProto request)
+ throws ServiceException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("deleteVolume called! volume name: " + request.getVolumeName()
+ + " force:" + request.getForce());
+ }
+ try {
+ if (request.hasForce()) {
+ impl.deleteVolume(request.getUserName(), request.getVolumeName(),
+ request.getForce());
+ } else {
+ impl.deleteVolume(request.getUserName(), request.getVolumeName(),
+ false);
+ }
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ return CBlockServiceProtocolProtos.DeleteVolumeResponseProto
+ .newBuilder().build();
+ }
+
+ @Override
+ public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume(
+ RpcController controller,
+ CBlockServiceProtocolProtos.InfoVolumeRequestProto request
+ ) throws ServiceException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("infoVolume called! volume name: " + request.getVolumeName());
+ }
+ CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp =
+ CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder();
+ CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto =
+ CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
+ VolumeInfo volumeInfo;
+ try {
+ volumeInfo = impl.infoVolume(request.getUserName(),
+ request.getVolumeName());
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+
+ volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize());
+ volumeInfoProto.setBlockSize(volumeInfo.getBlockSize());
+ volumeInfoProto.setUsage(volumeInfo.getUsage());
+ volumeInfoProto.setUserName(volumeInfo.getUserName());
+ volumeInfoProto.setVolumeName(volumeInfo.getVolumeName());
+ resp.setVolumeInfo(volumeInfoProto);
+ return resp.build();
+ }
+
+ @Override
+ public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume(
+ RpcController controller,
+ CBlockServiceProtocolProtos.ListVolumeRequestProto request
+ ) throws ServiceException {
+ CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp =
+ CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder();
+ String userName = null;
+ if (request.hasUserName()) {
+ userName = request.getUserName();
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("list volume received for :" + userName);
+ }
+ List<VolumeInfo> volumes;
+ try {
+ volumes = impl.listVolume(userName);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ for (VolumeInfo volume : volumes) {
+ CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto
+ = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
+ volumeEntryProto.setUserName(volume.getUserName());
+ volumeEntryProto.setVolumeName(volume.getVolumeName());
+ volumeEntryProto.setVolumeSize(volume.getVolumeSize());
+ volumeEntryProto.setBlockSize(volume.getBlockSize());
+ resp.addVolumeEntry(volumeEntryProto.build());
+ }
+ return resp.build();
+ }
+
+ public CBlockServiceProtocolServerSideTranslatorPB(
+ CBlockServiceProtocol impl) {
+ this.impl = impl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
new file mode 100644
index 0000000..5e03a92
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
@@ -0,0 +1,18 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.protocolPB;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
deleted file mode 100644
index 85f8d6f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
deleted file mode 100644
index 93ed005..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/DynamicProvisioner.java
+++ /dev/null
@@ -1,330 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.kubernetes;
-
-import com.google.gson.reflect.TypeToken;
-import com.squareup.okhttp.RequestBody;
-import io.kubernetes.client.ApiClient;
-import io.kubernetes.client.ApiException;
-import io.kubernetes.client.Configuration;
-import io.kubernetes.client.apis.CoreV1Api;
-import io.kubernetes.client.models.V1ISCSIVolumeSource;
-import io.kubernetes.client.models.V1ObjectMeta;
-import io.kubernetes.client.models.V1ObjectReference;
-import io.kubernetes.client.models.V1PersistentVolume;
-import io.kubernetes.client.models.V1PersistentVolumeClaim;
-import io.kubernetes.client.models.V1PersistentVolumeSpec;
-import io.kubernetes.client.util.Config;
-import io.kubernetes.client.util.Watch;
-import okio.Buffer;
-import org.apache.hadoop.cblock.cli.CBlockCli;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.storage.StorageManager;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.ratis.shaded.com.google.common.annotations.VisibleForTesting;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CBLOCK_USER;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY;
-
-/**
- * Kubernetes Dynamic Persistent Volume provisioner.
- *
- * Listens on the kubernetes feed and creates the appropriate cblock AND
- * kubernetes PersistentVolume according to the created PersistentVolumeClaims.
- */
-public class DynamicProvisioner implements Runnable{
-
- protected static final Logger LOGGER =
- LoggerFactory.getLogger(DynamicProvisioner.class);
-
- private static final String STORAGE_CLASS = "cblock";
-
- private static final String PROVISIONER_ID = "hadoop.apache.org/cblock";
- private static final String KUBERNETES_PROVISIONER_KEY =
- "volume.beta.kubernetes.io/storage-provisioner";
- private static final String KUBERNETES_BIND_COMPLETED_KEY =
- "pv.kubernetes.io/bind-completed";
-
- private boolean running = true;
-
- private final StorageManager storageManager;
-
- private String kubernetesConfigFile;
-
- private String externalIp;
-
- private int externalPort;
-
- private String cblockUser;
-
- private CoreV1Api api;
-
- private ApiClient client;
-
- private Thread watcherThread;
-
- public DynamicProvisioner(OzoneConfiguration ozoneConf,
- StorageManager storageManager) throws IOException {
- this.storageManager = storageManager;
-
- kubernetesConfigFile = ozoneConf
- .getTrimmed(DFS_CBLOCK_KUBERNETES_CONFIG_FILE_KEY);
-
- String jscsiServerAddress = ozoneConf
- .get(DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
- DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
-
- externalIp = ozoneConf.
- getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
-
- externalPort = ozoneConf.
- getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
- DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
-
- cblockUser = ozoneConf.getTrimmed(DFS_CBLOCK_KUBERNETES_CBLOCK_USER,
- DFS_CBLOCK_KUBERNETES_CBLOCK_USER_DEFAULT);
-
-
- }
-
- public void init() throws IOException {
- if (kubernetesConfigFile != null) {
- client = Config.fromConfig(kubernetesConfigFile);
- } else {
- client = Config.fromCluster();
- }
- client.getHttpClient().setReadTimeout(60, TimeUnit.SECONDS);
- Configuration.setDefaultApiClient(client);
- api = new CoreV1Api();
-
- watcherThread = new Thread(this);
- watcherThread.setName("DynamicProvisioner");
- watcherThread.setDaemon(true);
- }
-
- @Override
- public void run() {
- LOGGER.info("Starting kubernetes dynamic provisioner.");
- while (running) {
- String resourceVersion = null;
- try {
-
- Watch<V1PersistentVolumeClaim> watch = Watch.createWatch(client,
- api.listPersistentVolumeClaimForAllNamespacesCall(null,
- null,
- false,
- null,
- null,
- null,
- resourceVersion,
- null,
- true,
- null,
- null),
- new TypeToken<Watch.Response<V1PersistentVolumeClaim>>() {
- }.getType());
-
-
- //check the new pvc resources, and create cblock + pv if needed
- for (Watch.Response<V1PersistentVolumeClaim> item : watch) {
- V1PersistentVolumeClaim claim = item.object;
-
- if (isPvMissingForPvc(claim)) {
-
- LOGGER.info("Provisioning volumes for PVC {}/{}",
- claim.getMetadata().getNamespace(),
- claim.getMetadata().getName());
-
- if (LOGGER.isDebugEnabled()) {
- RequestBody request =
- api.getApiClient().serialize(claim, "application/json");
-
- final Buffer buffer = new Buffer();
- request.writeTo(buffer);
- LOGGER.debug("New PVC is detected: " + buffer.readUtf8());
- }
-
- String volumeName = createVolumeName(claim);
-
- long size = CBlockCli.parseSize(
- claim.getSpec().getResources().getRequests().get("storage"));
-
- createCBlock(volumeName, size);
- createPersistentVolumeFromPVC(item.object, volumeName);
- }
- }
- } catch (Exception ex) {
- if (ex.getCause() != null && ex
- .getCause() instanceof SocketTimeoutException) {
- //This is normal. We are connection to the kubernetes server and the
- //connection should be reopened time to time...
- LOGGER.debug("Time exception occured", ex);
- } else {
- LOGGER.error("Error on provisioning persistent volumes.", ex);
- try {
- //we can try again in the main loop
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- LOGGER.error("Error on sleeping after an error.", e);
- }
- }
- }
- }
- }
-
- private boolean isPvMissingForPvc(V1PersistentVolumeClaim claim) {
-
- Map<String, String> annotations = claim.getMetadata().getAnnotations();
-
- return claim.getStatus().getPhase().equals("Pending") && STORAGE_CLASS
- .equals(claim.getSpec().getStorageClassName()) && PROVISIONER_ID
- .equals(annotations.get(KUBERNETES_PROVISIONER_KEY)) && !"yes"
- .equals(annotations.get(KUBERNETES_BIND_COMPLETED_KEY));
- }
-
- @VisibleForTesting
- protected String createVolumeName(V1PersistentVolumeClaim claim) {
- return claim.getMetadata().getName() + "-" + claim.getMetadata()
- .getUid();
- }
-
- public void stop() {
- running = false;
- try {
- watcherThread.join(60000);
- } catch (InterruptedException e) {
- LOGGER.error("Kubernetes watcher thread can't stopped gracefully.", e);
- }
- }
-
- private void createCBlock(String volumeName, long size)
- throws CBlockException {
-
- MountVolumeResponse mountVolumeResponse =
- storageManager.isVolumeValid(cblockUser, volumeName);
- if (!mountVolumeResponse.getIsValid()) {
- storageManager
- .createVolume(cblockUser, volumeName, size, 4 * 1024);
- }
- }
-
- private void createPersistentVolumeFromPVC(V1PersistentVolumeClaim claim,
- String volumeName) throws ApiException, IOException {
-
- V1PersistentVolume v1PersistentVolume =
- persitenceVolumeBuilder(claim, volumeName);
-
- if (LOGGER.isDebugEnabled()) {
- RequestBody request =
- api.getApiClient().serialize(v1PersistentVolume, "application/json");
-
- final Buffer buffer = new Buffer();
- request.writeTo(buffer);
- LOGGER.debug("Creating new PV: " + buffer.readUtf8());
- }
- api.createPersistentVolume(v1PersistentVolume, null);
- }
-
- protected V1PersistentVolume persitenceVolumeBuilder(
- V1PersistentVolumeClaim claim,
- String volumeName) {
-
- V1PersistentVolume v1PersistentVolume = new V1PersistentVolume();
- v1PersistentVolume.setKind("PersistentVolume");
- v1PersistentVolume.setApiVersion("v1");
-
- V1ObjectMeta metadata = new V1ObjectMeta();
- metadata.setName(volumeName);
- metadata.setNamespace(claim.getMetadata().getNamespace());
- metadata.setAnnotations(new HashMap<>());
-
- metadata.getAnnotations()
- .put("pv.kubernetes.io/provisioned-by", PROVISIONER_ID);
-
- metadata.getAnnotations()
- .put("volume.beta.kubernetes.io/storage-class", STORAGE_CLASS);
-
- v1PersistentVolume.setMetadata(metadata);
-
- V1PersistentVolumeSpec spec = new V1PersistentVolumeSpec();
-
- spec.setCapacity(new HashMap<>());
- spec.getCapacity().put("storage",
- claim.getSpec().getResources().getRequests().get("storage"));
-
- spec.setAccessModes(new ArrayList<>());
- spec.getAccessModes().add("ReadWriteOnce");
-
- V1ObjectReference claimRef = new V1ObjectReference();
- claimRef.setName(claim.getMetadata().getName());
- claimRef.setNamespace(claim.getMetadata().getNamespace());
- claimRef.setKind(claim.getKind());
- claimRef.setApiVersion(claim.getApiVersion());
- claimRef.setUid(claim.getMetadata().getUid());
- spec.setClaimRef(claimRef);
-
- spec.persistentVolumeReclaimPolicy("Delete");
-
- V1ISCSIVolumeSource iscsi = new V1ISCSIVolumeSource();
- iscsi.setIqn(cblockUser + ":" + volumeName);
- iscsi.setLun(0);
- iscsi.setFsType("ext4");
- String portal = externalIp + ":" + externalPort;
- iscsi.setTargetPortal(portal);
- iscsi.setPortals(new ArrayList<>());
- iscsi.getPortals().add(portal);
-
- spec.iscsi(iscsi);
- v1PersistentVolume.setSpec(spec);
- return v1PersistentVolume;
- }
-
-
- @VisibleForTesting
- protected CoreV1Api getApi() {
- return api;
- }
-
- public void start() {
- watcherThread.start();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
deleted file mode 100644
index 3ec5aab..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/kubernetes/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * This package contains helper classes to run hadoop cluster in kubernetes
- * environment.
- */
-package org.apache.hadoop.cblock.kubernetes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
deleted file mode 100644
index 00064a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/ContainerDescriptor.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- *
- * The internal representation of a container maintained by CBlock server.
- * Include enough information to exactly identify a container for read/write
- * operation.
- *
- * NOTE that this class is work-in-progress. Depends on HDFS-7240 container
- * implementation. Currently only to allow testing.
- */
-public class ContainerDescriptor {
- private final String containerID;
- // the index of this container with in a volume
- // on creation, there may be no way to know the index of the container
- // as it is a volume specific information
- private int containerIndex;
- private Pipeline pipeline;
-
- public ContainerDescriptor(String containerID) {
- this.containerID = containerID;
- }
-
- public ContainerDescriptor(String containerID, int containerIndex) {
- this.containerID = containerID;
- this.containerIndex = containerIndex;
- }
-
- public void setContainerIndex(int idx) {
- this.containerIndex = idx;
- }
-
- public String getContainerID() {
- return containerID;
- }
-
- public void setPipeline(Pipeline pipeline) {
- this.pipeline = pipeline;
- }
-
- public Pipeline getPipeline() {
- return pipeline;
- }
-
- public int getContainerIndex() {
- return containerIndex;
- }
-
- public long getUtilization() {
- return 0;
- }
-
- public CBlockClientServerProtocolProtos.ContainerIDProto toProtobuf() {
- CBlockClientServerProtocolProtos.ContainerIDProto.Builder builder =
- CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
- builder.setContainerID(containerID);
- builder.setIndex(containerIndex);
- if (pipeline != null) {
- builder.setPipeline(pipeline.getProtobufMessage());
- }
- return builder.build();
- }
-
- public static ContainerDescriptor fromProtobuf(byte[] data)
- throws InvalidProtocolBufferException {
- CBlockClientServerProtocolProtos.ContainerIDProto id =
- CBlockClientServerProtocolProtos.ContainerIDProto.parseFrom(data);
- return new ContainerDescriptor(id.getContainerID(),
- (int)id.getIndex());
- }
-
- @Override
- public int hashCode() {
- return containerID.hashCode()*37 + containerIndex;
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof ContainerDescriptor) {
- ContainerDescriptor other = (ContainerDescriptor)o;
- return containerID.equals(other.containerID) &&
- containerIndex == other.containerIndex;
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
deleted file mode 100644
index 4f5930d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeDescriptor.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-/**
- * The internal representation maintained by CBlock server as the info for
- * a volume. Contains the list of containers belonging to this volume.
- *
- * Many methods of this class is made such that the volume information (
- * including container list) can be easily transformed into a Json string
- * that can be stored/parsed from a persistent store for cblock server
- * persistence.
- *
- * This class is still work-in-progress.
- */
-public class VolumeDescriptor {
- // The main data structure is the container location map
- // other thing are mainly just information
-
- // since only one operation at a time is allowed, no
- // need to consider concurrency control here
-
- // key is container id
-
- private static final Logger LOG =
- LoggerFactory.getLogger(VolumeDescriptor.class);
-
- private ConcurrentHashMap<String, ContainerDescriptor> containerMap;
- private String userName;
- private int blockSize;
- private long volumeSize;
- private String volumeName;
- // this is essentially the ordered keys of containerMap
- // which is kind of redundant information. But since we
- // are likely to access it frequently based on ordering.
- // keeping this copy to avoid having to sort the key every
- // time
- private List<String> containerIdOrdered;
-
- /**
- * This is not being called explicitly, but this is necessary as
- * it will be called by the parse method implicitly when
- * reconstructing the object from json string. The get*() methods
- * and set*() methods are for the same purpose also.
- */
- public VolumeDescriptor() {
- this(null, null, 0, 0);
- }
-
- public VolumeDescriptor(String userName, String volumeName, long volumeSize,
- int blockSize) {
- this.containerMap = new ConcurrentHashMap<>();
- this.userName = userName;
- this.volumeName = volumeName;
- this.blockSize = blockSize;
- this.volumeSize = volumeSize;
- this.containerIdOrdered = new LinkedList<>();
- }
-
- public String getUserName() {
- return userName;
- }
-
- public void setUserName(String userName) {
- this.userName = userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public void setVolumeName(String volumeName) {
- this.volumeName = volumeName;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public void setVolumeSize(long volumeSize) {
- this.volumeSize = volumeSize;
- }
-
- public int getBlockSize() {
- return blockSize;
- }
-
- public void setBlockSize(int blockSize) {
- this.blockSize = blockSize;
- }
-
- public void setContainerIDs(ArrayList<String> containerIDs) {
- containerIdOrdered.addAll(containerIDs);
- }
-
- public void addContainer(ContainerDescriptor containerDescriptor) {
- containerMap.put(containerDescriptor.getContainerID(),
- containerDescriptor);
- }
-
-
- public HashMap<String, Pipeline> getPipelines() {
- HashMap<String, Pipeline> pipelines = new HashMap<>();
- for (Map.Entry<String, ContainerDescriptor> entry :
- containerMap.entrySet()) {
- pipelines.put(entry.getKey(), entry.getValue().getPipeline());
- }
- return pipelines;
- }
-
- public boolean isEmpty() {
- VolumeInfo info = getInfo();
- return info.getUsage() == 0;
- }
-
- public VolumeInfo getInfo() {
- // TODO : need to actually go through all containers of this volume and
- // ask for their utilization.
- long utilization = 0;
- for (Map.Entry<String, ContainerDescriptor> entry :
- containerMap.entrySet()) {
- utilization += entry.getValue().getUtilization();
- }
- return new VolumeInfo(this.userName, this.volumeName,
- this.volumeSize, this.blockSize,
- utilization * blockSize);
- }
-
- public String[] getContainerIDs() {
- //ArrayList<Long> ids = new ArrayList(containerMap.keySet());
- //return ids.toArray(new Long[ids.size()]);
- return containerIdOrdered.toArray(new String[containerIdOrdered.size()]);
- }
-
- public List<String> getContainerIDsList() {
- return new ArrayList<>(containerIdOrdered);
- }
-
- public List<Pipeline> getContainerPipelines() {
- Map<String, Pipeline> tmp = getPipelines();
- List<Pipeline> pipelineList = new LinkedList<>();
- for (String containerIDString : containerIdOrdered) {
- pipelineList.add(tmp.get(containerIDString));
- }
- return pipelineList;
- }
-
- @Override
- public String toString() {
- String string = "";
- string += "Username:" + userName + "\n";
- string += "VolumeName:" + volumeName + "\n";
- string += "VolumeSize:" + volumeSize + "\n";
- string += "blockSize:" + blockSize + "\n";
- string += "containerIds:" + containerIdOrdered + "\n";
- string += "containerIdsWithObject:" + containerMap.keySet();
- return string;
- }
-
- public CBlockClientServerProtocolProtos.MountVolumeResponseProto
- toProtobuf() {
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder volume =
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.newBuilder();
- volume.setIsValid(true);
- volume.setVolumeName(volumeName);
- volume.setUserName(userName);
- volume.setVolumeSize(volumeSize);
- volume.setBlockSize(blockSize);
- for (String containerIDString : containerIdOrdered) {
- ContainerDescriptor containerDescriptor = containerMap.get(
- containerIDString);
- volume.addAllContainerIDs(containerDescriptor.toProtobuf());
- }
- return volume.build();
- }
-
- public static VolumeDescriptor fromProtobuf(byte[] data)
- throws InvalidProtocolBufferException {
- CBlockClientServerProtocolProtos.MountVolumeResponseProto volume =
- CBlockClientServerProtocolProtos.MountVolumeResponseProto
- .parseFrom(data);
- String userName = volume.getUserName();
- String volumeName = volume.getVolumeName();
- long volumeSize = volume.getVolumeSize();
- int blockSize = volume.getBlockSize();
- VolumeDescriptor volumeDescriptor = new VolumeDescriptor(userName,
- volumeName, volumeSize, blockSize);
- List<CBlockClientServerProtocolProtos.ContainerIDProto> containers
- = volume.getAllContainerIDsList();
-
- String[] containerOrdering = new String[containers.size()];
-
- for (CBlockClientServerProtocolProtos.ContainerIDProto containerProto :
- containers) {
- ContainerDescriptor containerDescriptor = new ContainerDescriptor(
- containerProto.getContainerID(),
- (int)containerProto.getIndex());
- if(containerProto.hasPipeline()) {
- containerDescriptor.setPipeline(
- Pipeline.getFromProtoBuf(containerProto.getPipeline()));
- }
- volumeDescriptor.addContainer(containerDescriptor);
- containerOrdering[containerDescriptor.getContainerIndex()] =
- containerDescriptor.getContainerID();
- }
- volumeDescriptor.setContainerIDs(
- new ArrayList<>(Arrays.asList(containerOrdering)));
- return volumeDescriptor;
- }
-
- @Override
- public int hashCode() {
- return userName.hashCode()*37 + volumeName.hashCode();
- }
-
- @Override
- public boolean equals(Object o) {
- if (o != null && o instanceof VolumeDescriptor) {
- VolumeDescriptor other = (VolumeDescriptor)o;
- if (!userName.equals(other.getUserName()) ||
- !volumeName.equals(other.getVolumeName()) ||
- volumeSize != other.getVolumeSize() ||
- blockSize != other.getBlockSize()) {
- return false;
- }
- if (containerIdOrdered.size() != other.containerIdOrdered.size() ||
- containerMap.size() != other.containerMap.size()) {
- return false;
- }
- for (int i = 0; i<containerIdOrdered.size(); i++) {
- if (!containerIdOrdered.get(i).equals(
- other.containerIdOrdered.get(i))) {
- return false;
- }
- }
- return containerMap.equals(other.containerMap);
- }
- return false;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
deleted file mode 100644
index 7f50c41..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/VolumeInfo.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
-
-/**
- * A wrapper class that represents the information about a volume. Used in
- * communication between CBlock client and CBlock server only.
- */
-public class VolumeInfo {
- private final String userName;
- private final String volumeName;
- private final long volumeSize;
- private final long blockSize;
- private final long usage;
-
- public VolumeInfo(String userName, String volumeName, long volumeSize,
- long blockSize, long usage) {
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.usage = usage;
- }
-
- // When listing volume, the usage will not be set.
- public VolumeInfo(String userName, String volumeName, long volumeSize,
- long blockSize) {
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.usage = -1;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public long getBlockSize() {
- return blockSize;
- }
-
- public long getUsage() {
- return usage;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- @Override
- public String toString() {
- return " userName:" + userName +
- " volumeName:" + volumeName +
- " volumeSize:" + volumeSize +
- " blockSize:" + blockSize +
- " (sizeInBlocks:" + volumeSize/blockSize + ")" +
- " usageInBlocks:" + usage;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
deleted file mode 100644
index a331d7a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/meta/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.meta;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
deleted file mode 100644
index a7d5d8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
deleted file mode 100644
index fc40cef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockClientProtocol.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * The protocol that CBlock client side uses to talk to server side. CBlock
- * client is the point where a volume is mounted. All the actual volume IO
- * operations will go through CBlock client after the volume is mounted.
- *
- * When users mount a volume on CBlock client, CBlock client side uses this
- * protocol to send mount request to CBlock server.
- */
-public interface CBlockClientProtocol {
- MountVolumeResponse mountVolume(String userName, String volumeName)
- throws IOException;
-
- List<VolumeInfo> listVolumes() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
deleted file mode 100644
index bf00bc0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/CBlockServiceProtocol.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * CBlock uses a separate command line tool to send volume management
- * operations to CBlock server, including create/delete/info/list volumes. This
- * is the protocol used by the command line tool to send these requests and get
- * responses from CBlock server.
- */
-@InterfaceAudience.Private
-public interface CBlockServiceProtocol {
-
- void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws IOException;
-
- void deleteVolume(String userName, String volumeName,
- boolean force) throws IOException;
-
- VolumeInfo infoVolume(String userName,
- String volumeName) throws IOException;
-
- List<VolumeInfo> listVolume(String userName) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
deleted file mode 100644
index 70ccd90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/MountVolumeResponse.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
-
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.util.HashMap;
-import java.util.List;
-
-/**
- * The response message of mounting a volume. Including enough information
- * for the client to communicate (perform IO) with the volume containers
- * directly.
- */
-public class MountVolumeResponse {
- private final boolean isValid;
- private final String userName;
- private final String volumeName;
- private final long volumeSize;
- private final int blockSize;
- private List<Pipeline> containerList;
- private HashMap<String, Pipeline> pipelineMap;
-
- public MountVolumeResponse(boolean isValid, String userName,
- String volumeName, long volumeSize, int blockSize,
- List<Pipeline> containerList,
- HashMap<String, Pipeline> pipelineMap) {
- this.isValid = isValid;
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.containerList = containerList;
- this.pipelineMap = pipelineMap;
- }
-
- public boolean getIsValid() {
- return isValid;
- }
-
- public String getUserName() {
- return userName;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public long getVolumeSize() {
- return volumeSize;
- }
-
- public int getBlockSize() {
- return blockSize;
- }
-
- public List<Pipeline> getContainerList() {
- return containerList;
- }
-
- public HashMap<String, Pipeline> getPipelineMap() {
- return pipelineMap;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
deleted file mode 100644
index 33438ec..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/proto/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.proto;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
deleted file mode 100644
index 99f3110..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolPB.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * This is the protocol CBlock client uses to talk to CBlock server.
- * CBlock client is the mounting point of a volume. When a user mounts a
- * volume, the cBlock client running on the local node will use this protocol
- * to talk to CBlock server to mount the volume.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockClientServerProtocolPB extends
- CBlockClientServerProtocolProtos
- .CBlockClientServerProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index bfe2130..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockClientServerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto.CBlockClientServerProtocolProtos;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * The server side implementation of cblock client to server protocol.
- */
-@InterfaceAudience.Private
-public class CBlockClientServerProtocolServerSideTranslatorPB implements
- CBlockClientServerProtocolPB {
-
- private final CBlockClientProtocol impl;
-
- public CBlockClientServerProtocolServerSideTranslatorPB(
- CBlockClientProtocol impl) {
- this.impl = impl;
- }
-
- @Override
- public CBlockClientServerProtocolProtos.MountVolumeResponseProto mountVolume(
- RpcController controller,
- CBlockClientServerProtocolProtos.MountVolumeRequestProto request)
- throws ServiceException {
- String userName = request.getUserName();
- String volumeName = request.getVolumeName();
- CBlockClientServerProtocolProtos.MountVolumeResponseProto.Builder
- resp =
- CBlockClientServerProtocolProtos
- .MountVolumeResponseProto.newBuilder();
- try {
- MountVolumeResponse result = impl.mountVolume(userName, volumeName);
- boolean isValid = result.getIsValid();
- resp.setIsValid(isValid);
- if (isValid) {
- resp.setUserName(result.getUserName());
- resp.setVolumeName(result.getVolumeName());
- resp.setVolumeSize(result.getVolumeSize());
- resp.setBlockSize(result.getBlockSize());
- List<Pipeline> containers = result.getContainerList();
- HashMap<String, Pipeline> pipelineMap = result.getPipelineMap();
-
- for (int i=0; i<containers.size(); i++) {
- CBlockClientServerProtocolProtos.ContainerIDProto.Builder id =
- CBlockClientServerProtocolProtos.ContainerIDProto.newBuilder();
- String containerName = containers.get(i).getContainerName();
- id.setContainerID(containerName);
- id.setIndex(i);
- if (pipelineMap.containsKey(containerName)) {
- id.setPipeline(pipelineMap.get(containerName).getProtobufMessage());
- }
- resp.addAllContainerIDs(id.build());
- }
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return resp.build();
- }
-
- @Override
- public CBlockClientServerProtocolProtos.ListVolumesResponseProto listVolumes(
- RpcController controller,
- CBlockClientServerProtocolProtos.ListVolumesRequestProto request)
- throws ServiceException {
- try {
- CBlockClientServerProtocolProtos.ListVolumesResponseProto.Builder resp =
- CBlockClientServerProtocolProtos.ListVolumesResponseProto
- .newBuilder();
- List<VolumeInfo> volumeInfos = impl.listVolumes();
- List<CBlockServiceProtocolProtos.VolumeInfoProto> convertedInfos =
- volumeInfos.stream().map(
- volumeInfo -> CBlockServiceProtocolProtos.VolumeInfoProto
- .newBuilder().setUserName(volumeInfo.getUserName())
- .setBlockSize(volumeInfo.getBlockSize())
- .setVolumeName(volumeInfo.getVolumeName())
- .setVolumeSize(volumeInfo.getVolumeSize())
- .setUsage(volumeInfo.getUsage()).build())
- .collect(Collectors.toList());
- resp.addAllVolumeEntry(convertedInfos);
- return resp.build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
deleted file mode 100644
index 282d6cd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolPB.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtocolInfo;
-
-/**
- * Users use a independent command line tool to talk to CBlock server for
- * volume operations (create/delete/info/list). This is the protocol used by
- * the the command line tool to send these requests to CBlock server.
- */
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocol",
- protocolVersion = 1)
-@InterfaceAudience.Private
-public interface CBlockServiceProtocolPB extends
- CBlockServiceProtocolProtos.CBlockServiceProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 8924a0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/CBlockServiceProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT;
-
-/**
- * Server side implementation of the protobuf service.
- */
-@InterfaceAudience.Private
-public class CBlockServiceProtocolServerSideTranslatorPB
- implements CBlockServiceProtocolPB {
-
- private final CBlockServiceProtocol impl;
- private static final Logger LOG =
- LoggerFactory.getLogger(
- CBlockServiceProtocolServerSideTranslatorPB.class);
-
- @Override
- public CBlockServiceProtocolProtos.CreateVolumeResponseProto createVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.CreateVolumeRequestProto request)
- throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("createVolume called! volume size: " + request.getVolumeSize()
- + " block size: " + request.getBlockSize());
- }
- try {
- if (request.hasBlockSize()) {
- impl.createVolume(request.getUserName(), request.getVolumeName(),
- request.getVolumeSize(), request.getBlockSize());
- } else{
- impl.createVolume(request.getUserName(), request.getVolumeName(),
- request.getVolumeSize(), DFS_CBLOCK_SERVICE_BLOCK_SIZE_DEFAULT);
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return CBlockServiceProtocolProtos.CreateVolumeResponseProto
- .newBuilder().build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.DeleteVolumeResponseProto deleteVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.DeleteVolumeRequestProto request)
- throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("deleteVolume called! volume name: " + request.getVolumeName()
- + " force:" + request.getForce());
- }
- try {
- if (request.hasForce()) {
- impl.deleteVolume(request.getUserName(), request.getVolumeName(),
- request.getForce());
- } else {
- impl.deleteVolume(request.getUserName(), request.getVolumeName(),
- false);
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- return CBlockServiceProtocolProtos.DeleteVolumeResponseProto
- .newBuilder().build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.InfoVolumeResponseProto infoVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.InfoVolumeRequestProto request
- ) throws ServiceException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("infoVolume called! volume name: " + request.getVolumeName());
- }
- CBlockServiceProtocolProtos.InfoVolumeResponseProto.Builder resp =
- CBlockServiceProtocolProtos.InfoVolumeResponseProto.newBuilder();
- CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeInfoProto =
- CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
- VolumeInfo volumeInfo;
- try {
- volumeInfo = impl.infoVolume(request.getUserName(),
- request.getVolumeName());
- } catch (IOException e) {
- throw new ServiceException(e);
- }
-
- volumeInfoProto.setVolumeSize(volumeInfo.getVolumeSize());
- volumeInfoProto.setBlockSize(volumeInfo.getBlockSize());
- volumeInfoProto.setUsage(volumeInfo.getUsage());
- volumeInfoProto.setUserName(volumeInfo.getUserName());
- volumeInfoProto.setVolumeName(volumeInfo.getVolumeName());
- resp.setVolumeInfo(volumeInfoProto);
- return resp.build();
- }
-
- @Override
- public CBlockServiceProtocolProtos.ListVolumeResponseProto listVolume(
- RpcController controller,
- CBlockServiceProtocolProtos.ListVolumeRequestProto request
- ) throws ServiceException {
- CBlockServiceProtocolProtos.ListVolumeResponseProto.Builder resp =
- CBlockServiceProtocolProtos.ListVolumeResponseProto.newBuilder();
- String userName = null;
- if (request.hasUserName()) {
- userName = request.getUserName();
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("list volume received for :" + userName);
- }
- List<VolumeInfo> volumes;
- try {
- volumes = impl.listVolume(userName);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- for (VolumeInfo volume : volumes) {
- CBlockServiceProtocolProtos.VolumeInfoProto.Builder volumeEntryProto
- = CBlockServiceProtocolProtos.VolumeInfoProto.newBuilder();
- volumeEntryProto.setUserName(volume.getUserName());
- volumeEntryProto.setVolumeName(volume.getVolumeName());
- volumeEntryProto.setVolumeSize(volume.getVolumeSize());
- volumeEntryProto.setBlockSize(volume.getBlockSize());
- resp.addVolumeEntry(volumeEntryProto.build());
- }
- return resp.build();
- }
-
- public CBlockServiceProtocolServerSideTranslatorPB(
- CBlockServiceProtocol impl) {
- this.impl = impl;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
deleted file mode 100644
index 5e03a92..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/protocolPB/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.protocolPB;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
deleted file mode 100644
index 865f3b2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/StorageManager.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.meta.ContainerDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeDescriptor;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.scm.client.ScmClient;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * This class maintains the key space of CBlock, more specifically, the
- * volume to container mapping. The core data structure
- * is a map from users to their volumes info, where volume info is a handler
- * to a volume, containing information for IO on that volume and a storage
- * client responsible for talking to the SCM.
- */
-public class StorageManager {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(StorageManager.class);
- private final ScmClient storageClient;
- private final int numThreads;
- private static final int MAX_THREADS =
- Runtime.getRuntime().availableProcessors() * 2;
- private static final int MAX_QUEUE_CAPACITY = 1024;
- private final String cblockId;
-
- /**
- * We will NOT have the situation where same kv pair getting
- * processed, but it is possible to have multiple kv pair being
- * processed at same time.
- *
- * So using just ConcurrentHashMap should be sufficient
- *
- * Again since currently same user accessing from multiple places
- * is not allowed, no need to consider concurrency of volume map
- * within one user
- */
- private ConcurrentHashMap<String, HashMap<String, VolumeDescriptor>>
- user2VolumeMap;
- // size of an underlying container.
- // TODO : assuming all containers are of the same size
- private long containerSizeB;
-
- public StorageManager(ScmClient storageClient,
- OzoneConfiguration ozoneConfig, String cblockId) throws IOException {
- this.storageClient = storageClient;
- this.user2VolumeMap = new ConcurrentHashMap<>();
- this.containerSizeB = storageClient.getContainerSize(null);
- this.numThreads =
- ozoneConfig.getInt(CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE,
- CBlockConfigKeys.DFS_CBLOCK_MANAGER_POOL_SIZE_DEFAULT);
- this.cblockId = cblockId;
- }
-
- /**
- * This call will put the volume into in-memory map.
- *
- * more specifically, make the volume discoverable on jSCSI server
- * and keep it's reference in-memory for look up.
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume,
- * @param volume a {@link VolumeDescriptor} object encapsulating the
- * information about the volume.
- */
- private void makeVolumeReady(String userName, String volumeName,
- VolumeDescriptor volume) {
- HashMap<String, VolumeDescriptor> userVolumes;
- if (user2VolumeMap.containsKey(userName)) {
- userVolumes = user2VolumeMap.get(userName);
- } else {
- userVolumes = new HashMap<>();
- user2VolumeMap.put(userName, userVolumes);
- }
- userVolumes.put(volumeName, volume);
- }
-
- /**
- * Called by CBlockManager to add volumes read from persistent store into
- * memory, need to contact SCM to setup the reference to the containers given
- * their id.
- *
- * Only for failover process where container meta info is read from
- * persistent store, and containers themselves are alive.
- *
- * TODO : Currently, this method is not being called as failover process
- * is not implemented yet.
- *
- * @param volumeDescriptor a {@link VolumeDescriptor} object encapsulating
- * the information about a volume.
- * @throws IOException when adding the volume failed. e.g. volume already
- * exist, or no more container available.
- */
- public synchronized void addVolume(VolumeDescriptor volumeDescriptor)
- throws IOException{
- String userName = volumeDescriptor.getUserName();
- String volumeName = volumeDescriptor.getVolumeName();
- LOGGER.info("addVolume:" + userName + ":" + volumeName);
- if (user2VolumeMap.containsKey(userName)
- && user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Volume already exist for "
- + userName + ":" + volumeName);
- }
- // the container ids are read from levelDB, setting up the
- // container handlers here.
- String[] containerIds = volumeDescriptor.getContainerIDs();
-
- for (String containerId : containerIds) {
- try {
- Pipeline pipeline = storageClient.getContainer(containerId);
- ContainerDescriptor containerDescriptor =
- new ContainerDescriptor(containerId);
- containerDescriptor.setPipeline(pipeline);
- volumeDescriptor.addContainer(containerDescriptor);
- } catch (IOException e) {
- LOGGER.error("Getting container failed! Container:{} error:{}",
- containerId, e);
- throw e;
- }
- }
- // now ready to put into in-memory map.
- makeVolumeReady(userName, volumeName, volumeDescriptor);
- }
-
- private class CreateContainerTask implements Runnable {
- private final VolumeDescriptor volume;
- private final int containerIdx;
- private final ArrayList<String> containerIds;
- private final AtomicInteger numFailed;
-
- CreateContainerTask(VolumeDescriptor volume, int containerIdx,
- ArrayList<String> containerIds,
- AtomicInteger numFailed) {
- this.volume = volume;
- this.containerIdx = containerIdx;
- this.containerIds = containerIds;
- this.numFailed = numFailed;
- }
-
- /**
- * When an object implementing interface <code>Runnable</code> is used
- * to create a thread, starting the thread causes the object's
- * <code>run</code> method to be called in that separately executing
- * thread.
- * <p>
- * The general contract of the method <code>run</code> is that it may
- * take any action whatsoever.
- *
- * @see Thread#run()
- */
- public void run() {
- ContainerDescriptor container = null;
- try {
- Pipeline pipeline = storageClient.createContainer(
- OzoneProtos.ReplicationType.STAND_ALONE,
- OzoneProtos.ReplicationFactor.ONE,
- KeyUtil.getContainerName(volume.getUserName(),
- volume.getVolumeName(), containerIdx), cblockId);
-
- container = new ContainerDescriptor(pipeline.getContainerName());
-
- container.setPipeline(pipeline);
- container.setContainerIndex(containerIdx);
- volume.addContainer(container);
- containerIds.set(containerIdx, container.getContainerID());
- } catch (Exception e) {
- numFailed.incrementAndGet();
- if (container != null) {
- LOGGER.error("Error creating container Container:{}:" +
- " index:{} error:{}", container.getContainerID(),
- containerIdx, e);
- } else {
- LOGGER.error("Error creating container.", e);
- }
- }
- }
- }
-
- private boolean createVolumeContainers(VolumeDescriptor volume) {
- ArrayList<String> containerIds = new ArrayList<>();
- ThreadPoolExecutor executor = new ThreadPoolExecutor(
- Math.min(numThreads, MAX_THREADS),
- MAX_THREADS, 1, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
- new ThreadPoolExecutor.CallerRunsPolicy());
-
- AtomicInteger numFailedCreates = new AtomicInteger(0);
- long allocatedSize = 0;
- int containerIdx = 0;
- while (allocatedSize < volume.getVolumeSize()) {
- // adding null to allocate space in ArrayList
- containerIds.add(containerIdx, null);
- Runnable task = new CreateContainerTask(volume, containerIdx,
- containerIds, numFailedCreates);
- executor.submit(task);
- allocatedSize += containerSizeB;
- containerIdx += 1;
- }
-
- // issue the command and then wait for it to finish
- executor.shutdown();
- try {
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
- } catch (InterruptedException e) {
- LOGGER.error("Error creating volume:{} error:{}",
- volume.getVolumeName(), e);
- executor.shutdownNow();
- Thread.currentThread().interrupt();
- }
-
- volume.setContainerIDs(containerIds);
- return numFailedCreates.get() == 0;
- }
-
- private void deleteContainer(String containerID, boolean force) {
- try {
- Pipeline pipeline = storageClient.getContainer(containerID);
- storageClient.deleteContainer(pipeline, force);
- } catch (Exception e) {
- LOGGER.error("Error deleting container Container:{} error:{}",
- containerID, e);
- }
- }
-
- private void deleteVolumeContainers(List<String> containers, boolean force)
- throws CBlockException {
- ThreadPoolExecutor executor = new ThreadPoolExecutor(
- Math.min(numThreads, MAX_THREADS),
- MAX_THREADS, 1, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(MAX_QUEUE_CAPACITY),
- new ThreadPoolExecutor.CallerRunsPolicy());
-
- for (String deleteContainer : containers) {
- if (deleteContainer != null) {
- Runnable task = () -> deleteContainer(deleteContainer, force);
- executor.submit(task);
- }
- }
-
- // issue the command and then wait for it to finish
- executor.shutdown();
- try {
- executor.awaitTermination(Long.MAX_VALUE, TimeUnit.MINUTES);
- } catch (InterruptedException e) {
- LOGGER.error("Error deleting containers error:{}", e);
- executor.shutdownNow();
- Thread.currentThread().interrupt();
- }
- }
-
- /**
- * Called by CBlock server when creating a fresh volume. The core
- * logic is adding needed information into in-memory meta data.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @param volumeSize the size of the volume.
- * @param blockSize the block size of the volume.
- * @throws CBlockException when the volume can not be created.
- */
- public synchronized void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws CBlockException {
- LOGGER.debug("createVolume:" + userName + ":" + volumeName);
- if (user2VolumeMap.containsKey(userName)
- && user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Volume already exist for "
- + userName + ":" + volumeName);
- }
- if (volumeSize < blockSize) {
- throw new CBlockException("Volume size smaller than block size? " +
- "volume size:" + volumeSize + " block size:" + blockSize);
- }
- VolumeDescriptor volume
- = new VolumeDescriptor(userName, volumeName, volumeSize, blockSize);
- boolean success = createVolumeContainers(volume);
- if (!success) {
- // cleanup the containers and throw the exception
- deleteVolumeContainers(volume.getContainerIDsList(), true);
- throw new CBlockException("Error when creating volume:" + volumeName);
- }
- makeVolumeReady(userName, volumeName, volume);
- }
-
- /**
- * Called by CBlock server to delete a specific volume. Mainly
- * to check whether it can be deleted, and remove it from in-memory meta
- * data.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @param force if set to false, only delete volume it is empty, otherwise
- * throw exception. if set to true, delete regardless.
- * @throws CBlockException when the volume can not be deleted.
- */
- public synchronized void deleteVolume(String userName, String volumeName,
- boolean force) throws CBlockException {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Deleting non-exist volume "
- + userName + ":" + volumeName);
- }
- if (!force && !user2VolumeMap.get(userName).get(volumeName).isEmpty()) {
- throw new CBlockException("Deleting a non-empty volume without force!");
- }
- VolumeDescriptor volume = user2VolumeMap.get(userName).remove(volumeName);
- deleteVolumeContainers(volume.getContainerIDsList(), force);
- if (user2VolumeMap.get(userName).size() == 0) {
- user2VolumeMap.remove(userName);
- }
- }
-
- /**
- * Called by CBlock server to get information of a specific volume.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume.
- * @return a {@link VolumeInfo} object encapsulating the information of the
- * volume.
- * @throws CBlockException when the information can not be retrieved.
- */
- public synchronized VolumeInfo infoVolume(String userName, String volumeName)
- throws CBlockException {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- throw new CBlockException("Getting info for non-exist volume "
- + userName + ":" + volumeName);
- }
- return user2VolumeMap.get(userName).get(volumeName).getInfo();
- }
-
- /**
- * Called by CBlock server to check whether the given volume can be
- * mounted, i.e. whether it can be found in the meta data.
- *
- * return a {@link MountVolumeResponse} with isValid flag to indicate
- * whether the volume can be mounted or not.
- *
- * @param userName the user name of the volume.
- * @param volumeName the name of the volume
- * @return a {@link MountVolumeResponse} object encapsulating whether the
- * volume is valid, and if yes, the requried information for client to
- * read/write the volume.
- */
- public synchronized MountVolumeResponse isVolumeValid(
- String userName, String volumeName) {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- // in the case of invalid volume, no need to set any value other than
- // isValid flag.
- return new MountVolumeResponse(false, null, null, 0, 0, null, null);
- }
- VolumeDescriptor volume = user2VolumeMap.get(userName).get(volumeName);
- return new MountVolumeResponse(true, userName,
- volumeName, volume.getVolumeSize(), volume.getBlockSize(),
- volume.getContainerPipelines(), volume.getPipelines());
- }
-
- /**
- * Called by CBlock manager to list all volumes.
- *
- * @param userName the userName whose volume to be listed, if set to null,
- * all volumes will be listed.
- * @return a list of {@link VolumeDescriptor} representing all volumes
- * requested.
- */
- public synchronized List<VolumeDescriptor> getAllVolume(String userName) {
- ArrayList<VolumeDescriptor> allVolumes = new ArrayList<>();
- if (userName == null) {
- for (Map.Entry<String, HashMap<String, VolumeDescriptor>> entry
- : user2VolumeMap.entrySet()) {
- allVolumes.addAll(entry.getValue().values());
- }
- } else {
- if (user2VolumeMap.containsKey(userName)) {
- allVolumes.addAll(user2VolumeMap.get(userName).values());
- }
- }
- return allVolumes;
- }
-
- /**
- * Only for testing the behavior of create/delete volumes.
- */
- @VisibleForTesting
- public VolumeDescriptor getVolume(String userName, String volumeName) {
- if (!user2VolumeMap.containsKey(userName)
- || !user2VolumeMap.get(userName).containsKey(volumeName)) {
- return null;
- }
- return user2VolumeMap.get(userName).get(volumeName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
deleted file mode 100644
index 4426e6d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/storage/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.storage;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
deleted file mode 100644
index beb9e32..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/KeyUtil.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
-
-/**
- * A simply class that generates key mappings. (e.g. from (userName, volumeName)
- * pair to a single string volumeKey.
- */
-public final class KeyUtil {
- public static String getVolumeKey(String userName, String volumeName) {
- return userName + ":" + volumeName;
- }
-
- public static String getContainerName(String userName, String volumeName,
- int containerID) {
- return getVolumeKey(userName, volumeName) + "#" + containerID;
- }
-
- public static String getUserNameFromVolumeKey(String key) {
- return key.split(":")[0];
- }
-
- public static String getVolumeFromVolumeKey(String key) {
- return key.split(":")[1];
- }
-
- public static boolean isValidVolumeKey(String key) {
- return key.contains(":");
- }
-
- private KeyUtil() {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
deleted file mode 100644
index 5b9aa0c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/util/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.util;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
index 72f39fc..4c94e38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSUtil.java
@@ -36,8 +36,6 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMESERVICE_ID;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYPASSWORD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_KEYSTORE_PASSWORD_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_SERVER_HTTPS_TRUSTSTORE_PASSWORD_KEY;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ENABLED_DEFAULT;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
@@ -73,7 +71,6 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -1497,23 +1494,6 @@ public class DFSUtil {
return password;
}
- public static boolean isOzoneEnabled(Configuration conf) {
- String securityEnabled = conf.get(CommonConfigurationKeysPublic
- .HADOOP_SECURITY_AUTHENTICATION,
- "simple");
- boolean securityAuthorizationEnabled = conf.getBoolean(
- CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION,
- false);
-
- if (securityEnabled.equals("kerberos") || securityAuthorizationEnabled) {
- LOG.error("Ozone is not supported in a security enabled cluster. ");
- return false;
- } else {
- return conf.getBoolean(OZONE_ENABLED,
- OZONE_ENABLED_DEFAULT);
- }
- }
-
/**
* Converts a Date into an ISO-8601 formatted datetime string.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
deleted file mode 100644
index eb18486..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/KeyManagerImpl.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.ozone.container.common.helpers.KeyUtils;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.utils.ContainerCache;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.utils.MetadataKeyFilters.KeyPrefixFilter;
-import org.apache.hadoop.utils.MetadataKeyFilters.MetadataKeyFilter;
-import org.apache.hadoop.utils.MetadataStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos
- .Result.NO_SUCH_KEY;
-
-/**
- * Key Manager impl.
- */
-public class KeyManagerImpl implements KeyManager {
- static final Logger LOG =
- LoggerFactory.getLogger(KeyManagerImpl.class);
-
- private static final float LOAD_FACTOR = 0.75f;
- private final ContainerManager containerManager;
- private final Configuration conf;
-
- /**
- * Constructs a key Manager.
- *
- * @param containerManager - Container Manager.
- */
- public KeyManagerImpl(ContainerManager containerManager, Configuration conf) {
- Preconditions.checkNotNull(containerManager, "Container manager cannot be" +
- " null");
- Preconditions.checkNotNull(conf, "Config cannot be null");
- this.containerManager = containerManager;
- this.conf = conf;
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void putKey(Pipeline pipeline, KeyData data) throws IOException {
- containerManager.readLock();
- try {
- // We are not locking the key manager since LevelDb serializes all actions
- // against a single DB. We rely on DB level locking to avoid conflicts.
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(containerName);
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- db.put(data.getKeyName().getBytes(KeyUtils.ENCODING), data
- .getProtoBufMessage().toByteArray());
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public KeyData getKey(KeyData data) throws IOException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(data, "Key data cannot be null");
- Preconditions.checkNotNull(data.getContainerName(),
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(data
- .getContainerName());
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
-
- byte[] kData = db.get(data.getKeyName().getBytes(KeyUtils.ENCODING));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- ContainerProtos.KeyData keyData =
- ContainerProtos.KeyData.parseFrom(kData);
- return KeyData.getFromProtoBuf(keyData);
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public void deleteKey(Pipeline pipeline, String keyName)
- throws IOException {
- containerManager.readLock();
- try {
- Preconditions.checkNotNull(pipeline, "Pipeline cannot be null");
- String containerName = pipeline.getContainerName();
- Preconditions.checkNotNull(containerName,
- "Container name cannot be null");
- ContainerData cData = containerManager.readContainer(containerName);
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- // This is a post condition that acts as a hint to the user.
- // Should never fail.
- Preconditions.checkNotNull(db, "DB cannot be null here");
- // Note : There is a race condition here, since get and delete
- // are not atomic. Leaving it here since the impact is refusing
- // to delete a key which might have just gotten inserted after
- // the get check.
-
- byte[] kData = db.get(keyName.getBytes(KeyUtils.ENCODING));
- if (kData == null) {
- throw new StorageContainerException("Unable to find the key.",
- NO_SUCH_KEY);
- }
- db.delete(keyName.getBytes(KeyUtils.ENCODING));
- } finally {
- containerManager.readUnlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- public List<KeyData> listKey(
- Pipeline pipeline, String prefix, String startKey, int count)
- throws IOException {
- Preconditions.checkNotNull(pipeline,
- "Pipeline cannot be null.");
- Preconditions.checkArgument(count > 0,
- "Count must be a positive number.");
- ContainerData cData = containerManager.readContainer(pipeline
- .getContainerName());
- MetadataStore db = KeyUtils.getDB(cData, conf);
-
- List<KeyData> result = new ArrayList<KeyData>();
- byte[] startKeyInBytes = startKey == null ? null :
- DFSUtil.string2Bytes(startKey);
- MetadataKeyFilter prefixFilter = new KeyPrefixFilter(prefix);
- List<Map.Entry<byte[], byte[]>> range =
- db.getSequentialRangeKVs(startKeyInBytes, count, prefixFilter);
- for (Map.Entry<byte[], byte[]> entry : range) {
- String keyName = KeyUtils.getKeyName(entry.getKey());
- KeyData value = KeyUtils.getKeyData(entry.getValue());
- KeyData data = new KeyData(value.getContainerName(), keyName);
- result.add(data);
- }
- return result;
- }
-
- /**
- * Shutdown keyManager.
- */
- @Override
- public void shutdown() {
- Preconditions.checkState(this.containerManager.hasWriteLock(), "asserts " +
- "that we are holding the container manager lock when shutting down.");
- KeyUtils.shutdownCache(ContainerCache.getInstance(conf));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 0123ab1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/RandomContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Randomly choosing containers for block deletion.
- */
-public class RandomContainerDeletionChoosingPolicy
- implements ContainerDeletionChoosingPolicy {
- private static final Logger LOG =
- LoggerFactory.getLogger(RandomContainerDeletionChoosingPolicy.class);
-
- @Override
- public List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException {
- Preconditions.checkNotNull(candidateContainers,
- "Internal assertion: candidate containers cannot be null");
-
- int currentCount = 0;
- List<ContainerData> result = new LinkedList<>();
- ContainerStatus[] values = new ContainerStatus[candidateContainers.size()];
- // to get a shuffle list
- for (ContainerStatus entry : DFSUtil.shuffle(
- candidateContainers.values().toArray(values))) {
- if (currentCount < count) {
- result.add(entry.getContainer());
- currentCount++;
-
- LOG.debug("Select container {} for block deletion, "
- + "pending deletion blocks num: {}.",
- entry.getContainer().getContainerName(),
- entry.getNumPendingDeletionBlocks());
- } else {
- break;
- }
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
deleted file mode 100644
index 7ef91a9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/StorageLocationReport.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- * Storage location stats of datanodes that provide back store for containers.
- *
- */
-public class StorageLocationReport {
- public static final StorageLocationReport[] EMPTY_ARRAY = {};
-
- private final String id;
- private final boolean failed;
- private final long capacity;
- private final long scmUsed;
- private final long remaining;
-
- public StorageLocationReport(String id, boolean failed,
- long capacity, long scmUsed, long remaining) {
- this.id = id;
- this.failed = failed;
- this.capacity = capacity;
- this.scmUsed = scmUsed;
- this.remaining = remaining;
- }
-
- public String getId() {
- return id;
- }
-
- public boolean isFailed() {
- return failed;
- }
-
- public long getCapacity() {
- return capacity;
- }
-
- public long getScmUsed() {
- return scmUsed;
- }
-
- public long getRemaining() {
- return remaining;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3f4cdaa..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/TopNOrderedContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDeletionChoosingPolicy;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.google.common.base.Preconditions;
-
-/**
- * TopN Ordered choosing policy that choosing containers based on pending
- * deletion blocks' number.
- */
-public class TopNOrderedContainerDeletionChoosingPolicy
- implements ContainerDeletionChoosingPolicy {
- private static final Logger LOG =
- LoggerFactory.getLogger(TopNOrderedContainerDeletionChoosingPolicy.class);
-
- /** customized comparator used to compare differentiate container status. **/
- private static final Comparator<ContainerStatus> CONTAINER_STATUS_COMPARATOR
- = new Comparator<ContainerStatus>() {
- @Override
- public int compare(ContainerStatus c1, ContainerStatus c2) {
- return Integer.compare(c2.getNumPendingDeletionBlocks(),
- c1.getNumPendingDeletionBlocks());
- }
- };
-
- @Override
- public List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException {
- Preconditions.checkNotNull(candidateContainers,
- "Internal assertion: candidate containers cannot be null");
-
- List<ContainerData> result = new LinkedList<>();
- List<ContainerStatus> orderedList = new LinkedList<>();
- orderedList.addAll(candidateContainers.values());
- Collections.sort(orderedList, CONTAINER_STATUS_COMPARATOR);
-
- // get top N list ordered by pending deletion blocks' number
- int currentCount = 0;
- for (ContainerStatus entry : orderedList) {
- if (currentCount < count) {
- if (entry.getNumPendingDeletionBlocks() > 0) {
- result.add(entry.getContainer());
- currentCount++;
-
- LOG.debug(
- "Select container {} for block deletion, "
- + "pending deletion blocks num: {}.",
- entry.getContainer().getContainerName(),
- entry.getNumPendingDeletionBlocks());
- } else {
- LOG.debug("Stop looking for next container, there is no"
- + " pending deletion block contained in remaining containers.");
- break;
- }
- } else {
- break;
- }
- }
-
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
deleted file mode 100644
index 16da5d9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/impl/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.impl;
-
-/**
- This package is contains Ozone container implementation.
-**/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
deleted file mode 100644
index c933924..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ChunkManager.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-/**
- * Chunk Manager allows read, write, delete and listing of chunks in
- * a container.
- */
-public interface ChunkManager {
-
- /**
- * writes a given chunk.
- * @param pipeline - Name and the set of machines that make this container.
- * @param keyName - Name of the Key.
- * @param info - ChunkInfo.
- * @param stage - Chunk Stage write.
- * @throws StorageContainerException
- */
- void writeChunk(Pipeline pipeline, String keyName,
- ChunkInfo info, byte[] data, ContainerProtos.Stage stage)
- throws StorageContainerException;
-
- /**
- * reads the data defined by a chunk.
- * @param pipeline - container pipeline.
- * @param keyName - Name of the Key
- * @param info - ChunkInfo.
- * @return byte array
- * @throws StorageContainerException
- *
- * TODO: Right now we do not support partial reads and writes of chunks.
- * TODO: Explore if we need to do that for ozone.
- */
- byte[] readChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
- StorageContainerException;
-
- /**
- * Deletes a given chunk.
- * @param pipeline - Pipeline.
- * @param keyName - Key Name
- * @param info - Chunk Info
- * @throws StorageContainerException
- */
- void deleteChunk(Pipeline pipeline, String keyName, ChunkInfo info) throws
- StorageContainerException;
-
- // TODO : Support list operations.
-
- /**
- * Shutdown the chunkManager.
- */
- void shutdown();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
deleted file mode 100644
index 3e0a283..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDeletionChoosingPolicy.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ContainerStatus;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-
-/**
- * This interface is used for choosing desired containers for
- * block deletion.
- */
-public interface ContainerDeletionChoosingPolicy {
-
- /**
- * Chooses desired containers for block deletion.
- * @param count
- * how many to return
- * @param candidateContainers
- * candidate containers collection
- * @return container data list
- * @throws StorageContainerException
- */
- List<ContainerData> chooseContainerForBlockDeletion(int count,
- Map<String, ContainerStatus> candidateContainers)
- throws StorageContainerException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
deleted file mode 100644
index 8aae004..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerDispatcher.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-
-/**
- * Dispatcher acts as the bridge between the transport layer and
- * the actual container layer. This layer is capable of transforming
- * protobuf objects into corresponding class and issue the function call
- * into the lower layers.
- *
- * The reply from the request is dispatched to the client.
- */
-public interface ContainerDispatcher {
- /**
- * Dispatches commands to container layer.
- * @param msg - Command Request
- * @return Command Response
- */
- ContainerCommandResponseProto dispatch(ContainerCommandRequestProto msg);
-
- /**
- * Initialize the Dispatcher.
- */
- void init();
-
- /**
- * Shutdown Dispatcher services.
- */
- void shutdown();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
deleted file mode 100644
index 9c5fcea..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManager.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
-
-import java.io.IOException;
-import java.nio.file.Path;
-
-/**
- * Returns physical path locations, where the containers will be created.
- */
-public interface ContainerLocationManager {
- /**
- * Returns the path where the container should be placed from a set of
- * locations.
- *
- * @return A path where we should place this container and metadata.
- * @throws IOException
- */
- Path getContainerPath() throws IOException;
-
- /**
- * Returns the path where the container Data file are stored.
- *
- * @return a path where we place the LevelDB and data files of a container.
- * @throws IOException
- */
- Path getDataPath(String containerName) throws IOException;
-
- /**
- * Returns an array of storage location usage report.
- * @return storage location usage report.
- */
- StorageLocationReport[] getLocationReport() throws IOException;
-
- /**
- * Supports clean shutdown of container.
- *
- * @throws IOException
- */
- void shutdown() throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
deleted file mode 100644
index 88e6148..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerLocationManagerMXBean.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.container.common.impl.StorageLocationReport;
-
-import java.io.IOException;
-
-/**
- * Returns physical path locations, where the containers will be created.
- */
-public interface ContainerLocationManagerMXBean {
-
- /**
- * Returns an array of storage location usage report.
- *
- * @return storage location usage report.
- */
- StorageLocationReport[] getLocationReport() throws IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
deleted file mode 100644
index 04afc3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerManager.java
+++ /dev/null
@@ -1,276 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.hdfs.util.RwLock;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.util.List;
-
-/**
- * Interface for container operations.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public interface ContainerManager extends RwLock {
-
- /**
- * Init call that sets up a container Manager.
- *
- * @param config - Configuration.
- * @param containerDirs - List of Metadata Container locations.
- * @param datanodeID - Datanode ID
- * @throws StorageContainerException
- */
- void init(Configuration config, List<StorageLocation> containerDirs,
- DatanodeID datanodeID) throws IOException;
-
- /**
- * Creates a container with the given name.
- *
- * @param pipeline -- Nodes which make up this container.
- * @param containerData - Container Name and metadata.
- * @throws StorageContainerException
- */
- void createContainer(Pipeline pipeline, ContainerData containerData)
- throws StorageContainerException;
-
- /**
- * Deletes an existing container.
- *
- * @param pipeline - nodes that make this container.
- * @param containerName - name of the container.
- * @param forceDelete - whether this container should be deleted forcibly.
- * @throws StorageContainerException
- */
- void deleteContainer(Pipeline pipeline, String containerName,
- boolean forceDelete) throws StorageContainerException;
-
- /**
- * Update an existing container.
- *
- * @param pipeline container nodes
- * @param containerName name of the container
- * @param data container data
- * @param forceUpdate if true, update container forcibly.
- * @throws StorageContainerException
- */
- void updateContainer(Pipeline pipeline, String containerName,
- ContainerData data, boolean forceUpdate) throws StorageContainerException;
-
- /**
- * As simple interface for container Iterations.
- *
- * @param prefix - Return only values matching this prefix
- * @param count - how many to return
- * @param prevKey - Previous key - Server returns results from this point.
- * @param data - Actual containerData
- * @throws StorageContainerException
- */
- void listContainer(String prefix, long count, String prevKey,
- List<ContainerData> data)
- throws StorageContainerException;
-
- /**
- * Choose containers for block deletion.
- *
- * @param count - how many to return
- * @throws StorageContainerException
- */
- List<ContainerData> chooseContainerForBlockDeletion(int count)
- throws StorageContainerException;
-
- /**
- * Get metadata about a specific container.
- *
- * @param containerName - Name of the container
- * @return ContainerData - Container Data.
- * @throws StorageContainerException
- */
- ContainerData readContainer(String containerName)
- throws StorageContainerException;
-
- /**
- * Closes a open container, if it is already closed or does not exist a
- * StorageContainerException is thrown.
- * @param containerName - Name of the container.
- * @throws StorageContainerException
- */
- void closeContainer(String containerName)
- throws StorageContainerException, NoSuchAlgorithmException;
-
- /**
- * Checks if a container exists.
- * @param containerName - Name of the container.
- * @return true if the container is open false otherwise.
- * @throws StorageContainerException - Throws Exception if we are not
- * able to find the container.
- */
- boolean isOpen(String containerName) throws StorageContainerException;
-
- /**
- * Supports clean shutdown of container.
- *
- * @throws StorageContainerException
- */
- void shutdown() throws IOException;
-
- /**
- * Sets the Chunk Manager.
- *
- * @param chunkManager - ChunkManager.
- */
- void setChunkManager(ChunkManager chunkManager);
-
- /**
- * Gets the Chunk Manager.
- *
- * @return ChunkManager.
- */
- ChunkManager getChunkManager();
-
- /**
- * Sets the Key Manager.
- *
- * @param keyManager - Key Manager.
- */
- void setKeyManager(KeyManager keyManager);
-
- /**
- * Gets the Key Manager.
- *
- * @return KeyManager.
- */
- KeyManager getKeyManager();
-
- /**
- * Get the Node Report of container storage usage.
- * @return node report.
- */
- SCMNodeReport getNodeReport() throws IOException;
-
- /**
- * Gets container report.
- * @return container report.
- * @throws IOException
- */
- ContainerReportsRequestProto getContainerReport() throws IOException;
-
- /**
- * Gets container reports.
- * @return List of all closed containers.
- * @throws IOException
- */
- List<ContainerData> getContainerReports() throws IOException;
-
- /**
- * Increase pending deletion blocks count number of specified container.
- *
- * @param numBlocks
- * increment count number
- * @param containerId
- * container id
- */
- void incrPendingDeletionBlocks(int numBlocks, String containerId);
-
- /**
- * Decrease pending deletion blocks count number of specified container.
- *
- * @param numBlocks
- * decrement count number
- * @param containerId
- * container id
- */
- void decrPendingDeletionBlocks(int numBlocks, String containerId);
-
- /**
- * Increase the read count of the container.
- * @param containerName - Name of the container.
- */
- void incrReadCount(String containerName);
-
- /**
- * Increse the read counter for bytes read from the container.
- * @param containerName - Name of the container.
- * @param readBytes - bytes read from the container.
- */
- void incrReadBytes(String containerName, long readBytes);
-
-
- /**
- * Increase the write count of the container.
- * @param containerName - Name of the container.
- */
- void incrWriteCount(String containerName);
-
- /**
- * Increase the write counter for bytes write into the container.
- * @param containerName - Name of the container.
- * @param writeBytes - bytes write into the container.
- */
- void incrWriteBytes(String containerName, long writeBytes);
-
- /**
- * Increase the bytes used by the container.
- * @param containerName - Name of the container.
- * @param used - additional bytes used by the container.
- * @return the current bytes used.
- */
- long incrBytesUsed(String containerName, long used);
-
- /**
- * Decrease the bytes used by the container.
- * @param containerName - Name of the container.
- * @param used - additional bytes reclaimed by the container.
- * @return the current bytes used.
- */
- long decrBytesUsed(String containerName, long used);
-
- /**
- * Get the bytes used by the container.
- * @param containerName - Name of the container.
- * @return the current bytes used by the container.
- */
- long getBytesUsed(String containerName);
-
- /**
- * Get the number of keys in the container.
- * @param containerName - Name of the container.
- * @return the current key count.
- */
- long getNumKeys(String containerName);
-
- /**
- * Get the container report state to send via HB to SCM.
- * @return container report state.
- */
- ReportState getContainerReportState();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
deleted file mode 100644
index 7d2ba0a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/ContainerReportManager.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-
-/**
- * Interface for container report manager operations.
- */
-public interface ContainerReportManager {
-
- /**
- * Get the container report state.
- * @return the container report state.
- */
- ReportState getContainerReportState();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
deleted file mode 100644
index a613d2a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/KeyManager.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-
-import org.apache.hadoop.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.ozone.container.common.helpers.KeyData;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * KeyManager deals with Key Operations in the container Level.
- */
-public interface KeyManager {
- /**
- * Puts or overwrites a key.
- *
- * @param pipeline - Pipeline.
- * @param data - Key Data.
- * @throws IOException
- */
- void putKey(Pipeline pipeline, KeyData data) throws IOException;
-
- /**
- * Gets an existing key.
- *
- * @param data - Key Data.
- * @return Key Data.
- * @throws IOException
- */
- KeyData getKey(KeyData data) throws IOException;
-
- /**
- * Deletes an existing Key.
- *
- * @param pipeline - Pipeline.
- * @param keyName Key Data.
- * @throws StorageContainerException
- */
- void deleteKey(Pipeline pipeline, String keyName)
- throws IOException;
-
- /**
- * List keys in a container.
- *
- * @param pipeline - pipeline.
- * @param prefix - Prefix in needed.
- * @param startKey - Key to start from, EMPTY_STRING to begin.
- * @param count - Number of keys to return.
- * @return List of Keys that match the criteria.
- */
- List<KeyData> listKey(Pipeline pipeline, String prefix, String startKey,
- int count) throws IOException;
-
- /**
- * Shutdown keyManager.
- */
- void shutdown();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
deleted file mode 100644
index d83bf95..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/interfaces/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.interfaces;
-/**
- This package contains common ozone container interfaces.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
deleted file mode 100644
index 1638a36..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/package-info.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common;
-/**
- Common Container Layer. At this layer the abstractions are:
-
- 1. Containers - Both data and metadata containers.
- 2. Keys - Key/Value pairs that live inside a container.
- 3. Chunks - Keys can be composed of many chunks.
-
- Ozone uses these abstractions to build Volumes, Buckets and Keys.
-
- **/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
deleted file mode 100644
index de55d96..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/DatanodeStateMachine.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler
- .CloseContainerHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.CommandDispatcher;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.ContainerReportHandler;
-import org.apache.hadoop.ozone.container.common.statemachine.commandhandler.DeleteBlocksCommandHandler;
-import org.apache.hadoop.ozone.container.ozoneimpl.OzoneContainer;
-import org.apache.hadoop.ozone.protocol.commands.SCMCommand;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicLong;
-
-/**
- * State Machine Class.
- */
-public class DatanodeStateMachine implements Closeable {
- @VisibleForTesting
- static final Logger LOG =
- LoggerFactory.getLogger(DatanodeStateMachine.class);
- private final ExecutorService executorService;
- private final Configuration conf;
- private final SCMConnectionManager connectionManager;
- private final long heartbeatFrequency;
- private StateContext context;
- private final OzoneContainer container;
- private DatanodeID datanodeID = null;
- private final CommandDispatcher commandDispatcher;
- private long commandsHandled;
- private AtomicLong nextHB;
- private Thread stateMachineThread = null;
- private Thread cmdProcessThread = null;
-
- /**
- * Constructs a a datanode state machine.
- *
- * @param datanodeID - DatanodeID used to identify a datanode
- * @param conf - Configuration.
- */
- public DatanodeStateMachine(DatanodeID datanodeID,
- Configuration conf) throws IOException {
- this.conf = conf;
- executorService = HadoopExecutors.newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Datanode State Machine Thread - %d").build());
- connectionManager = new SCMConnectionManager(conf);
- context = new StateContext(this.conf, DatanodeStates.getInitState(), this);
- heartbeatFrequency = TimeUnit.SECONDS.toMillis(
- OzoneClientUtils.getScmHeartbeatInterval(conf));
- container = new OzoneContainer(datanodeID, new OzoneConfiguration(conf));
- this.datanodeID = datanodeID;
- nextHB = new AtomicLong(Time.monotonicNow());
-
- // When we add new handlers just adding a new handler here should do the
- // trick.
- commandDispatcher = CommandDispatcher.newBuilder()
- .addHandler(new ContainerReportHandler())
- .addHandler(new CloseContainerHandler())
- .addHandler(new DeleteBlocksCommandHandler(
- container.getContainerManager(), conf))
- .setConnectionManager(connectionManager)
- .setContainer(container)
- .setContext(context)
- .build();
- }
-
- public void setDatanodeID(DatanodeID datanodeID) {
- this.datanodeID = datanodeID;
- }
-
- /**
- *
- * Return DatanodeID if set, return null otherwise.
- *
- * @return datanodeID
- */
- public DatanodeID getDatanodeID() {
- return this.datanodeID;
- }
-
- /**
- * Returns the Connection manager for this state machine.
- *
- * @return - SCMConnectionManager.
- */
- public SCMConnectionManager getConnectionManager() {
- return connectionManager;
- }
-
- public OzoneContainer getContainer() {
- return this.container;
- }
-
- /**
- * Runs the state machine at a fixed frequency.
- */
- private void start() throws IOException {
- long now = 0;
-
- container.start();
- initCommandHandlerThread(conf);
- while (context.getState() != DatanodeStates.SHUTDOWN) {
- try {
- LOG.debug("Executing cycle Number : {}", context.getExecutionCount());
- nextHB.set(Time.monotonicNow() + heartbeatFrequency);
- context.setReportState(container.getNodeReport());
- context.setContainerReportState(container.getContainerReportState());
- context.execute(executorService, heartbeatFrequency,
- TimeUnit.MILLISECONDS);
- now = Time.monotonicNow();
- if (now < nextHB.get()) {
- Thread.sleep(nextHB.get() - now);
- }
- } catch (InterruptedException e) {
- // Ignore this exception.
- } catch (Exception e) {
- LOG.error("Unable to finish the execution.", e);
- }
- }
- }
-
- /**
- * Gets the current context.
- *
- * @return StateContext
- */
- public StateContext getContext() {
- return context;
- }
-
- /**
- * Sets the current context.
- *
- * @param context - Context
- */
- public void setContext(StateContext context) {
- this.context = context;
- }
-
- /**
- * Closes this stream and releases any system resources associated with it. If
- * the stream is already closed then invoking this method has no effect.
- * <p>
- * <p> As noted in {@link AutoCloseable#close()}, cases where the close may
- * fail require careful attention. It is strongly advised to relinquish the
- * underlying resources and to internally <em>mark</em> the {@code Closeable}
- * as closed, prior to throwing the {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- if (stateMachineThread != null) {
- stateMachineThread.interrupt();
- }
- if (cmdProcessThread != null) {
- cmdProcessThread.interrupt();
- }
- context.setState(DatanodeStates.getLastState());
- executorService.shutdown();
- try {
- if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
- executorService.shutdownNow();
- }
-
- if (!executorService.awaitTermination(5, TimeUnit.SECONDS)) {
- LOG.error("Unable to shutdown state machine properly.");
- }
- } catch (InterruptedException e) {
- LOG.error("Error attempting to shutdown.", e);
- executorService.shutdownNow();
- Thread.currentThread().interrupt();
- }
-
- if (connectionManager != null) {
- connectionManager.close();
- }
-
- if(container != null) {
- container.stop();
- }
- }
-
- /**
- * States that a datanode can be in. GetNextState will move this enum from
- * getInitState to getLastState.
- */
- public enum DatanodeStates {
- INIT(1),
- RUNNING(2),
- SHUTDOWN(3);
- private final int value;
-
- /**
- * Constructs ContainerStates.
- *
- * @param value Enum Value
- */
- DatanodeStates(int value) {
- this.value = value;
- }
-
- /**
- * Returns the first State.
- *
- * @return First State.
- */
- public static DatanodeStates getInitState() {
- return INIT;
- }
-
- /**
- * The last state of endpoint states.
- *
- * @return last state.
- */
- public static DatanodeStates getLastState() {
- return SHUTDOWN;
- }
-
- /**
- * returns the numeric value associated with the endPoint.
- *
- * @return int.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Returns the next logical state that endPoint should move to. This
- * function assumes the States are sequentially numbered.
- *
- * @return NextState.
- */
- public DatanodeStates getNextState() {
- if (this.value < getLastState().getValue()) {
- int stateValue = this.getValue() + 1;
- for (DatanodeStates iter : values()) {
- if (stateValue == iter.getValue()) {
- return iter;
- }
- }
- }
- return getLastState();
- }
- }
-
- /**
- * Start datanode state machine as a single thread daemon.
- */
- public void startDaemon() {
- Runnable startStateMachineTask = () -> {
- try {
- start();
- LOG.info("Ozone container server started.");
- } catch (Exception ex) {
- LOG.error("Unable to start the DatanodeState Machine", ex);
- }
- };
- stateMachineThread = new ThreadFactoryBuilder()
- .setDaemon(true)
- .setNameFormat("Datanode State Machine Thread - %d")
- .build().newThread(startStateMachineTask);
- stateMachineThread.start();
- }
-
- /**
- * Stop the daemon thread of the datanode state machine.
- */
- public synchronized void stopDaemon() {
- try {
- context.setState(DatanodeStates.SHUTDOWN);
- this.close();
- LOG.info("Ozone container server stopped.");
- } catch (IOException e) {
- LOG.error("Stop ozone container server failed.", e);
- }
- }
-
- /**
- *
- * Check if the datanode state machine daemon is stopped.
- *
- * @return True if datanode state machine daemon is stopped
- * and false otherwise.
- */
- @VisibleForTesting
- public boolean isDaemonStopped() {
- return this.executorService.isShutdown()
- && this.getContext().getExecutionCount() == 0
- && this.getContext().getState() == DatanodeStates.SHUTDOWN;
- }
-
- /**
- * Create a command handler thread.
- *
- * @param config
- */
- private void initCommandHandlerThread(Configuration config) {
-
- /**
- * Task that periodically checks if we have any outstanding commands.
- * It is assumed that commands can be processed slowly and in order.
- * This assumption might change in future. Right now due to this assumption
- * we have single command queue process thread.
- */
- Runnable processCommandQueue = () -> {
- long now;
- while (getContext().getState() != DatanodeStates.SHUTDOWN) {
- SCMCommand command = getContext().getNextCommand();
- if (command != null) {
- commandDispatcher.handle(command);
- commandsHandled++;
- } else {
- try {
- // Sleep till the next HB + 1 second.
- now = Time.monotonicNow();
- if (nextHB.get() > now) {
- Thread.sleep((nextHB.get() - now) + 1000L);
- }
- } catch (InterruptedException e) {
- // Ignore this exception.
- }
- }
- }
- };
-
- // We will have only one thread for command processing in a datanode.
- cmdProcessThread = getCommandHandlerThread(processCommandQueue);
- cmdProcessThread.start();
- }
-
- private Thread getCommandHandlerThread(Runnable processCommandQueue) {
- Thread handlerThread = new Thread(processCommandQueue);
- handlerThread.setDaemon(true);
- handlerThread.setName("Command processor thread");
- handlerThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
- // Let us just restart this thread after logging a critical error.
- // if this thread is not running we cannot handle commands from SCM.
- LOG.error("Critical Error : Command processor thread encountered an " +
- "error. Thread: {}", t.toString(), e);
- getCommandHandlerThread(processCommandQueue).start();
- });
- return handlerThread;
- }
-
- /**
- * Returns the number of commands handled by the datanode.
- * @return count
- */
- @VisibleForTesting
- public long getCommandHandled() {
- return commandsHandled;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
deleted file mode 100644
index 4f1f47f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachine.java
+++ /dev/null
@@ -1,293 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocolPB
- .StorageContainerDatanodeProtocolClientSideTranslatorPB;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.time.ZonedDateTime;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * Endpoint is used as holder class that keeps state around the RPC endpoint.
- */
-public class EndpointStateMachine
- implements Closeable, EndpointStateMachineMBean {
- static final Logger
- LOG = LoggerFactory.getLogger(EndpointStateMachine.class);
- private final StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint;
- private final AtomicLong missedCount;
- private final InetSocketAddress address;
- private final Lock lock;
- private final Configuration conf;
- private EndPointStates state;
- private VersionResponse version;
- private ZonedDateTime lastSuccessfulHeartbeat;
-
- /**
- * Constructs RPC Endpoints.
- *
- * @param endPoint - RPC endPoint.
- */
- public EndpointStateMachine(InetSocketAddress address,
- StorageContainerDatanodeProtocolClientSideTranslatorPB endPoint,
- Configuration conf) {
- this.endPoint = endPoint;
- this.missedCount = new AtomicLong(0);
- this.address = address;
- state = EndPointStates.getInitState();
- lock = new ReentrantLock();
- this.conf = conf;
- }
-
- /**
- * Takes a lock on this EndPoint so that other threads don't use this while we
- * are trying to communicate via this endpoint.
- */
- public void lock() {
- lock.lock();
- }
-
- /**
- * Unlocks this endpoint.
- */
- public void unlock() {
- lock.unlock();
- }
-
- /**
- * Returns the version that we read from the server if anyone asks .
- *
- * @return - Version Response.
- */
- public VersionResponse getVersion() {
- return version;
- }
-
- /**
- * Sets the Version reponse we recieved from the SCM.
- *
- * @param version VersionResponse
- */
- public void setVersion(VersionResponse version) {
- this.version = version;
- }
-
- /**
- * Returns the current State this end point is in.
- *
- * @return - getState.
- */
- public EndPointStates getState() {
- return state;
- }
-
- @Override
- public int getVersionNumber() {
- if (version != null) {
- return version.getProtobufMessage().getSoftwareVersion();
- } else {
- return -1;
- }
- }
-
- /**
- * Sets the endpoint state.
- *
- * @param epState - end point state.
- */
- public EndPointStates setState(EndPointStates epState) {
- this.state = epState;
- return this.state;
- }
-
- /**
- * Closes the connection.
- *
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- if (endPoint != null) {
- endPoint.close();
- }
- }
-
- /**
- * We maintain a count of how many times we missed communicating with a
- * specific SCM. This is not made atomic since the access to this is always
- * guarded by the read or write lock. That is, it is serialized.
- */
- public void incMissed() {
- this.missedCount.incrementAndGet();
- }
-
- /**
- * Returns the value of the missed count.
- *
- * @return int
- */
- public long getMissedCount() {
- return this.missedCount.get();
- }
-
- @Override
- public String getAddressString() {
- return getAddress().toString();
- }
-
- public void zeroMissedCount() {
- this.missedCount.set(0);
- }
-
- /**
- * Returns the InetAddress of the endPoint.
- *
- * @return - EndPoint.
- */
- public InetSocketAddress getAddress() {
- return this.address;
- }
-
- /**
- * Returns real RPC endPoint.
- *
- * @return rpc client.
- */
- public StorageContainerDatanodeProtocolClientSideTranslatorPB
- getEndPoint() {
- return endPoint;
- }
-
- /**
- * Returns the string that represents this endpoint.
- *
- * @return - String
- */
- public String toString() {
- return address.toString();
- }
-
- /**
- * Logs exception if needed.
- * @param ex - Exception
- */
- public void logIfNeeded(Exception ex) {
- LOG.trace("Incrementing the Missed count. Ex : {}", ex);
- this.incMissed();
- if (this.getMissedCount() % OzoneClientUtils.getLogWarnInterval(conf) ==
- 0) {
- LOG.warn("Unable to communicate to SCM server at {}. We have not been " +
- "able to communicate to this SCM server for past {} seconds.",
- this.getAddress().getHostString() + ":" + this.getAddress().getPort(),
- this.getMissedCount() * OzoneClientUtils.getScmHeartbeatInterval(
- this.conf));
- }
- }
-
-
- /**
- * States that an Endpoint can be in.
- * <p>
- * This is a sorted list of states that EndPoint will traverse.
- * <p>
- * GetNextState will move this enum from getInitState to getLastState.
- */
- public enum EndPointStates {
- GETVERSION(1),
- REGISTER(2),
- HEARTBEAT(3),
- SHUTDOWN(4); // if you add value after this please edit getLastState too.
- private final int value;
-
- /**
- * Constructs endPointStates.
- *
- * @param value state.
- */
- EndPointStates(int value) {
- this.value = value;
- }
-
- /**
- * Returns the first State.
- *
- * @return First State.
- */
- public static EndPointStates getInitState() {
- return GETVERSION;
- }
-
- /**
- * The last state of endpoint states.
- *
- * @return last state.
- */
- public static EndPointStates getLastState() {
- return SHUTDOWN;
- }
-
- /**
- * returns the numeric value associated with the endPoint.
- *
- * @return int.
- */
- public int getValue() {
- return value;
- }
-
- /**
- * Returns the next logical state that endPoint should move to.
- * The next state is computed by adding 1 to the current state.
- *
- * @return NextState.
- */
- public EndPointStates getNextState() {
- if (this.getValue() < getLastState().getValue()) {
- int stateValue = this.getValue() + 1;
- for (EndPointStates iter : values()) {
- if (stateValue == iter.getValue()) {
- return iter;
- }
- }
- }
- return getLastState();
- }
- }
-
- public long getLastSuccessfulHeartbeat() {
- return lastSuccessfulHeartbeat == null ?
- 0 :
- lastSuccessfulHeartbeat.toEpochSecond();
- }
-
- public void setLastSuccessfulHeartbeat(
- ZonedDateTime lastSuccessfulHeartbeat) {
- this.lastSuccessfulHeartbeat = lastSuccessfulHeartbeat;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
deleted file mode 100644
index 4f64bde..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/EndpointStateMachineMBean.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-
-/**
- * JMX representation of an EndpointStateMachine.
- */
-public interface EndpointStateMachineMBean {
-
- long getMissedCount();
-
- String getAddressString();
-
- EndpointStateMachine.EndPointStates getState();
-
- int getVersionNumber();
-
- long getLastSuccessfulHeartbeat();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
deleted file mode 100644
index a6767c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManager.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.metrics2.util.MBeans;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.ozone.protocolPB
- .StorageContainerDatanodeProtocolClientSideTranslatorPB;
-import org.apache.hadoop.ozone.protocolPB.StorageContainerDatanodeProtocolPB;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.management.ObjectName;
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.*;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-/**
- * SCMConnectionManager - Acts as a class that manages the membership
- * information of the SCMs that we are working with.
- */
-public class SCMConnectionManager
- implements Closeable, SCMConnectionManagerMXBean {
- private static final Logger LOG =
- LoggerFactory.getLogger(SCMConnectionManager.class);
-
- private final ReadWriteLock mapLock;
- private final Map<InetSocketAddress, EndpointStateMachine> scmMachines;
-
- private final int rpcTimeout;
- private final Configuration conf;
- private final ObjectName jmxBean;
-
- public SCMConnectionManager(Configuration conf) {
- this.mapLock = new ReentrantReadWriteLock();
- Long timeOut = OzoneClientUtils.getScmRpcTimeOutInMilliseconds(conf);
- this.rpcTimeout = timeOut.intValue();
- this.scmMachines = new HashMap<>();
- this.conf = conf;
- jmxBean = MBeans.register("OzoneDataNode",
- "SCMConnectionManager",
- this);
- }
-
-
- /**
- * Returns Config.
- *
- * @return ozoneConfig.
- */
- public Configuration getConf() {
- return conf;
- }
-
- /**
- * Get RpcTimeout.
- *
- * @return - Return RPC timeout.
- */
- public int getRpcTimeout() {
- return rpcTimeout;
- }
-
-
- /**
- * Takes a read lock.
- */
- public void readLock() {
- this.mapLock.readLock().lock();
- }
-
- /**
- * Releases the read lock.
- */
- public void readUnlock() {
- this.mapLock.readLock().unlock();
- }
-
- /**
- * Takes the write lock.
- */
- public void writeLock() {
- this.mapLock.writeLock().lock();
- }
-
- /**
- * Releases the write lock.
- */
- public void writeUnlock() {
- this.mapLock.writeLock().unlock();
- }
-
- /**
- * adds a new SCM machine to the target set.
- *
- * @param address - Address of the SCM machine to send heatbeat to.
- * @throws IOException
- */
- public void addSCMServer(InetSocketAddress address) throws IOException {
- writeLock();
- try {
- if (scmMachines.containsKey(address)) {
- LOG.warn("Trying to add an existing SCM Machine to Machines group. " +
- "Ignoring the request.");
- return;
- }
- RPC.setProtocolEngine(conf, StorageContainerDatanodeProtocolPB.class,
- ProtobufRpcEngine.class);
- long version =
- RPC.getProtocolVersion(StorageContainerDatanodeProtocolPB.class);
-
- StorageContainerDatanodeProtocolPB rpcProxy = RPC.getProxy(
- StorageContainerDatanodeProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf), getRpcTimeout());
-
- StorageContainerDatanodeProtocolClientSideTranslatorPB rpcClient =
- new StorageContainerDatanodeProtocolClientSideTranslatorPB(rpcProxy);
-
- EndpointStateMachine endPoint =
- new EndpointStateMachine(address, rpcClient, conf);
- scmMachines.put(address, endPoint);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Removes a SCM machine for the target set.
- *
- * @param address - Address of the SCM machine to send heatbeat to.
- * @throws IOException
- */
- public void removeSCMServer(InetSocketAddress address) throws IOException {
- writeLock();
- try {
- if (!scmMachines.containsKey(address)) {
- LOG.warn("Trying to remove a non-existent SCM machine. " +
- "Ignoring the request.");
- return;
- }
-
- EndpointStateMachine endPoint = scmMachines.get(address);
- endPoint.close();
- scmMachines.remove(address);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Returns all known RPCEndpoints.
- *
- * @return - List of RPC Endpoints.
- */
- public Collection<EndpointStateMachine> getValues() {
- return scmMachines.values();
- }
-
- @Override
- public void close() throws IOException {
- getValues().forEach(endpointStateMachine
- -> IOUtils.cleanupWithLogger(LOG, endpointStateMachine));
- MBeans.unregister(jmxBean);
- }
-
- @Override
- public List<EndpointStateMachineMBean> getSCMServers() {
- readLock();
- try {
- return Collections
- .unmodifiableList(new ArrayList<>(scmMachines.values()));
-
- } finally {
- readUnlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
deleted file mode 100644
index 25ef163..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/SCMConnectionManagerMXBean.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.statemachine;
-
-import java.util.List;
-
-/**
- * JMX information about the connected SCM servers.
- */
-public interface SCMConnectionManagerMXBean {
-
- List<EndpointStateMachineMBean> getSCMServers();
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
deleted file mode 100644
index 9371387..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/ContainerCloser.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-package org.apache.hadoop.ozone.scm.container.closer;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
-import org.apache.hadoop.ozone.protocol.commands.CloseContainerCommand;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.util.Time;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CONTAINER_REPORT_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT;
-
-/**
- * A class that manages closing of containers. This allows transition from a
- * open but full container to a closed container, to which no data is written.
- */
-public class ContainerCloser {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerCloser.class);
- private static final long MULTIPLIER = 3L;
- private static final int CLEANUP_WATER_MARK = 1000;
- private final NodeManager nodeManager;
- private final Map<String, Long> commandIssued;
- private final Configuration configuration;
- private final AtomicInteger mapCount;
- private final long reportInterval;
- private final AtomicInteger threadRunCount;
- private final AtomicBoolean isRunning;
-
- /**
- * Constructs the ContainerCloser class.
- *
- * @param nodeManager - NodeManager
- * @param conf - Configuration
- */
- public ContainerCloser(NodeManager nodeManager, Configuration conf) {
- Preconditions.checkNotNull(nodeManager);
- Preconditions.checkNotNull(conf);
- this.nodeManager = nodeManager;
- this.configuration = conf;
- this.commandIssued = new ConcurrentHashMap<>();
- this.mapCount = new AtomicInteger(0);
- this.threadRunCount = new AtomicInteger(0);
- this.isRunning = new AtomicBoolean(false);
- this.reportInterval = this.configuration.getTimeDuration(
- OZONE_CONTAINER_REPORT_INTERVAL,
- OZONE_CONTAINER_REPORT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
- Preconditions.checkState(this.reportInterval > 0,
- "report interval has to be greater than 0");
- }
-
- @VisibleForTesting
- public static int getCleanupWaterMark() {
- return CLEANUP_WATER_MARK;
- }
-
- /**
- * Sends a Container Close command to the data nodes where this container
- * lives.
- *
- * @param info - ContainerInfo.
- */
- public void close(OzoneProtos.SCMContainerInfo info) {
-
- if (commandIssued.containsKey(info.getContainerName())) {
- // We check if we issued a close command in last 3 * reportInterval secs.
- long commandQueueTime = commandIssued.get(info.getContainerName());
- long currentTime = TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow());
- if (currentTime > commandQueueTime + (MULTIPLIER * reportInterval)) {
- commandIssued.remove(info.getContainerName());
- mapCount.decrementAndGet();
- } else {
- // Ignore this request, since we just issued a close command. We
- // should wait instead of sending a command to datanode again.
- return;
- }
- }
-
- // if we reached here, it means that we have not issued a command to the
- // data node in last (3 times report interval). We are presuming that is
- // enough time to close the container. Let us go ahead and queue a close
- // to all the datanodes that participate in the container.
- //
- // Three important things to note here:
- //
- // 1. It is ok to send this command multiple times to a datanode. Close
- // container is an idempotent command, if the container is already closed
- // then we have no issues.
- //
- // 2. The container close command is issued to all datanodes. But
- // depending on the pipeline type, some of the datanodes might ignore it.
- //
- // 3. SCM will see that datanode is closed from container reports, but it
- // is possible that datanodes might get close commands since
- // this queue can be emptied by a datanode after a close report is send
- // to SCM. In that case also, data node will ignore this command.
-
- OzoneProtos.Pipeline pipeline = info.getPipeline();
- for (HdfsProtos.DatanodeIDProto datanodeID :
- pipeline.getPipelineChannel().getMembersList()) {
- nodeManager.addDatanodeCommand(DatanodeID.getFromProtoBuf(datanodeID),
- new CloseContainerCommand(info.getContainerName()));
- }
- if (!commandIssued.containsKey(info.getContainerName())) {
- commandIssued.put(info.getContainerName(),
- TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow()));
- mapCount.incrementAndGet();
- }
- // run the hash map cleaner thread if needed, non-blocking call.
- runCleanerThreadIfNeeded();
- }
-
- private void runCleanerThreadIfNeeded() {
- // Let us check if we should run a cleaner thread, not using map.size
- // since it runs a loop in the case of the concurrentMap.
- if (mapCount.get() > CLEANUP_WATER_MARK &&
- isRunning.compareAndSet(false, true)) {
- Runnable entryCleaner = () -> {
- LOG.debug("Starting close container Hash map cleaner.");
- try {
- for (Map.Entry<String, Long> entry : commandIssued.entrySet()) {
- long commandQueueTime = entry.getValue();
- if (commandQueueTime + (MULTIPLIER * reportInterval) >
- TimeUnit.MILLISECONDS.toSeconds(Time.monotonicNow())) {
-
- // It is possible for this remove to fail due to race conditions.
- // No big deal we will cleanup next time.
- commandIssued.remove(entry.getKey());
- mapCount.decrementAndGet();
- }
- }
- isRunning.compareAndSet(true, false);
- LOG.debug("Finished running, close container Hash map cleaner.");
- } catch (Exception ex) {
- LOG.error("Unable to finish cleaning the closed containers map.", ex);
- }
- };
-
- // Launch the cleaner thread when we need instead of having a daemon
- // thread that is sleeping all the time. We need to set the Daemon to
- // true to avoid blocking clean exits.
- Thread cleanerThread = new ThreadFactoryBuilder()
- .setDaemon(true)
- .setNameFormat("Closed Container Cleaner Thread - %d")
- .build().newThread(entryCleaner);
- threadRunCount.incrementAndGet();
- cleanerThread.start();
- }
- }
-
- @VisibleForTesting
- public int getThreadRunCount() {
- return threadRunCount.get();
- }
-
- @VisibleForTesting
- public int getCloseCount() {
- return mapCount.get();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
deleted file mode 100644
index 2d0f257..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/closer/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- *
- */
-
-/**
- * This package has class that close a container. That is move a container from
- * open state to close state.
- */
-package org.apache.hadoop.ozone.scm.container.closer;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
deleted file mode 100644
index 4ac490c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container;
-/**
- * This package contains routines to manage the container location and
- * mapping inside SCM
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
deleted file mode 100644
index 0cf1fde..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/ContainerPlacementPolicy.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * A ContainerPlacementPolicy support choosing datanodes to build replication
- * pipeline with specified constraints.
- */
-public interface ContainerPlacementPolicy {
-
- /**
- * Given the replication factor and size required, return set of datanodes
- * that satisfy the nodes and size requirement.
- * @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
- * @return list of datanodes chosen.
- * @throws IOException
- */
- List<DatanodeID> chooseDatanodes(int nodesRequired, long sizeRequired)
- throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
deleted file mode 100644
index b5a1a13..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMCommonPolicy.java
+++ /dev/null
@@ -1,194 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Random;
-import java.util.stream.Collectors;
-
-/**
- * SCM CommonPolicy implements a set of invariants which are common
- * for all container placement policies, acts as the repository of helper
- * functions which are common to placement policies.
- */
-public abstract class SCMCommonPolicy implements ContainerPlacementPolicy {
- @VisibleForTesting
- static final Logger LOG =
- LoggerFactory.getLogger(SCMCommonPolicy.class);
- private final NodeManager nodeManager;
- private final Random rand;
- private final Configuration conf;
-
- /**
- * Constructs SCM Common Policy Class.
- *
- * @param nodeManager NodeManager
- * @param conf Configuration class.
- */
- public SCMCommonPolicy(NodeManager nodeManager, Configuration conf) {
- this.nodeManager = nodeManager;
- this.rand = new Random();
- this.conf = conf;
- }
-
- /**
- * Return node manager.
- *
- * @return node manager
- */
- public NodeManager getNodeManager() {
- return nodeManager;
- }
-
- /**
- * Returns the Random Object.
- *
- * @return rand
- */
- public Random getRand() {
- return rand;
- }
-
- /**
- * Get Config.
- *
- * @return Configuration
- */
- public Configuration getConf() {
- return conf;
- }
-
- /**
- * Given the replication factor and size required, return set of datanodes
- * that satisfy the nodes and size requirement.
- * <p>
- * Here are some invariants of container placement.
- * <p>
- * 1. We place containers only on healthy nodes.
- * 2. We place containers on nodes with enough space for that container.
- * 3. if a set of containers are requested, we either meet the required
- * number of nodes or we fail that request.
- *
- * @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
- * @return list of datanodes chosen.
- * @throws SCMException SCM exception.
- */
-
- public List<DatanodeID> chooseDatanodes(int nodesRequired, final long
- sizeRequired) throws SCMException {
- List<DatanodeID> healthyNodes =
- nodeManager.getNodes(OzoneProtos.NodeState.HEALTHY);
- String msg;
- if (healthyNodes.size() == 0) {
- msg = "No healthy node found to allocate container.";
- LOG.error(msg);
- throw new SCMException(msg, SCMException.ResultCodes
- .FAILED_TO_FIND_HEALTHY_NODES);
- }
-
- if (healthyNodes.size() < nodesRequired) {
- msg = String.format("Not enough healthy nodes to allocate container. %d "
- + " datanodes required. Found %d",
- nodesRequired, healthyNodes.size());
- LOG.error(msg);
- throw new SCMException(msg,
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- List<DatanodeID> healthyList = healthyNodes.stream().filter(d ->
- hasEnoughSpace(d, sizeRequired)).collect(Collectors.toList());
-
- if (healthyList.size() < nodesRequired) {
- msg = String.format("Unable to find enough nodes that meet the space " +
- "requirement of %d bytes in healthy node set." +
- " Nodes required: %d Found: %d",
- sizeRequired, nodesRequired, healthyList.size());
- LOG.error(msg);
- throw new SCMException(msg,
- SCMException.ResultCodes.FAILED_TO_FIND_NODES_WITH_SPACE);
- }
-
- return healthyList;
- }
-
- /**
- * Returns true if this node has enough space to meet our requirement.
- *
- * @param datanodeID DatanodeID
- * @return true if we have enough space.
- */
- private boolean hasEnoughSpace(DatanodeID datanodeID, long sizeRequired) {
- SCMNodeMetric nodeMetric = nodeManager.getNodeStat(datanodeID);
- return (nodeMetric != null) && nodeMetric.get().getRemaining()
- .hasResources(sizeRequired);
- }
-
- /**
- * This function invokes the derived classes chooseNode Function to build a
- * list of nodes. Then it verifies that invoked policy was able to return
- * expected number of nodes.
- *
- * @param nodesRequired - Nodes Required
- * @param healthyNodes - List of Nodes in the result set.
- * @return List of Datanodes that can be used for placement.
- * @throws SCMException
- */
- public List<DatanodeID> getResultSet(int nodesRequired, List<DatanodeID>
- healthyNodes) throws SCMException {
- List<DatanodeID> results = new LinkedList<>();
- for (int x = 0; x < nodesRequired; x++) {
- // invoke the choose function defined in the derived classes.
- DatanodeID nodeId = chooseNode(healthyNodes);
- if (nodeId != null) {
- results.add(nodeId);
- }
- }
-
- if (results.size() < nodesRequired) {
- LOG.error("Unable to find the required number of healthy nodes that " +
- "meet the criteria. Required nodes: {}, Found nodes: {}",
- nodesRequired, results.size());
- throw new SCMException("Unable to find required number of nodes.",
- SCMException.ResultCodes.FAILED_TO_FIND_SUITABLE_NODE);
- }
- return results;
- }
-
- /**
- * Choose a datanode according to the policy, this function is implemented
- * by the actual policy class. For example, PlacementCapacity or
- * PlacementRandom.
- *
- * @param healthyNodes - Set of healthy nodes we can choose from.
- * @return DatanodeID
- */
- public abstract DatanodeID chooseNode(List<DatanodeID> healthyNodes);
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
deleted file mode 100644
index de9bb74..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementCapacity.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.container.placement.metrics.SCMNodeMetric;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Container placement policy that randomly choose datanodes with remaining
- * space to satisfy the size constraints.
- * <p>
- * The Algorithm is as follows, Pick 2 random nodes from a given pool of nodes
- * and then pick the node which lower utilization. This leads to a higher
- * probability of nodes with lower utilization to be picked.
- * <p>
- * For those wondering why we choose two nodes randomly and choose the node
- * with lower utilization. There are links to this original papers in
- * HDFS-11564.
- * <p>
- * A brief summary -- We treat the nodes from a scale of lowest utilized to
- * highest utilized, there are (s * ( s + 1)) / 2 possibilities to build
- * distinct pairs of nodes. There are s - k pairs of nodes in which the rank
- * k node is less than the couple. So probability of a picking a node is
- * (2 * (s -k)) / (s * (s - 1)).
- * <p>
- * In English, There is a much higher probability of picking less utilized nodes
- * as compared to nodes with higher utilization since we pick 2 nodes and
- * then pick the node with lower utilization.
- * <p>
- * This avoids the issue of users adding new nodes into the cluster and HDFS
- * sending all traffic to those nodes if we only use a capacity based
- * allocation scheme. Unless those nodes are part of the set of the first 2
- * nodes then newer nodes will not be in the running to get the container.
- * <p>
- * This leads to an I/O pattern where the lower utilized nodes are favoured
- * more than higher utilized nodes, but part of the I/O will still go to the
- * older higher utilized nodes.
- * <p>
- * With this algorithm in place, our hope is that balancer tool needs to do
- * little or no work and the cluster will achieve a balanced distribution
- * over time.
- */
-public final class SCMContainerPlacementCapacity extends SCMCommonPolicy {
- @VisibleForTesting
- static final Logger LOG =
- LoggerFactory.getLogger(SCMContainerPlacementCapacity.class);
-
- /**
- * Constructs a Container Placement with considering only capacity.
- * That is this policy tries to place containers based on node weight.
- *
- * @param nodeManager Node Manager
- * @param conf Configuration
- */
- public SCMContainerPlacementCapacity(final NodeManager nodeManager,
- final Configuration conf) {
- super(nodeManager, conf);
- }
-
- /**
- * Called by SCM to choose datanodes.
- *
- * @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
- * @return List of datanodes.
- * @throws SCMException SCMException
- */
- @Override
- public List<DatanodeID> chooseDatanodes(final int nodesRequired,
- final long sizeRequired) throws SCMException {
- List<DatanodeID> healthyNodes =
- super.chooseDatanodes(nodesRequired, sizeRequired);
- if (healthyNodes.size() == nodesRequired) {
- return healthyNodes;
- }
- return getResultSet(nodesRequired, healthyNodes);
- }
-
- /**
- * Find a node from the healthy list and return it after removing it from the
- * list that we are operating on.
- *
- * @param healthyNodes - List of healthy nodes that meet the size
- * requirement.
- * @return DatanodeID that is chosen.
- */
- @Override
- public DatanodeID chooseNode(List<DatanodeID> healthyNodes) {
- int firstNodeNdx = getRand().nextInt(healthyNodes.size());
- int secondNodeNdx = getRand().nextInt(healthyNodes.size());
-
- DatanodeID chosenID;
- // There is a possibility that both numbers will be same.
- // if that is so, we just return the node.
- if (firstNodeNdx == secondNodeNdx) {
- chosenID = healthyNodes.get(firstNodeNdx);
- } else {
- DatanodeID firstNodeID = healthyNodes.get(firstNodeNdx);
- DatanodeID secondNodeID = healthyNodes.get(secondNodeNdx);
- SCMNodeMetric firstNodeMetric =
- getNodeManager().getNodeStat(firstNodeID);
- SCMNodeMetric secondNodeMetric =
- getNodeManager().getNodeStat(secondNodeID);
- chosenID = firstNodeMetric.isGreater(secondNodeMetric.get())
- ? firstNodeID : secondNodeID;
- }
- healthyNodes.remove(chosenID);
- return chosenID;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
deleted file mode 100644
index b145b14..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/SCMContainerPlacementRandom.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-
-/**
- * Container placement policy that randomly chooses healthy datanodes.
- * This is very similar to current HDFS placement. That is we
- * just randomly place containers without any considerations of utilization.
- * <p>
- * That means we rely on balancer to achieve even distribution of data.
- * Balancer will need to support containers as a feature before this class
- * can be practically used.
- */
-public final class SCMContainerPlacementRandom extends SCMCommonPolicy
- implements ContainerPlacementPolicy {
- @VisibleForTesting
- static final Logger LOG =
- LoggerFactory.getLogger(SCMContainerPlacementRandom.class);
-
- /**
- * Construct a random Block Placement policy.
- *
- * @param nodeManager nodeManager
- * @param conf Config
- */
- public SCMContainerPlacementRandom(final NodeManager nodeManager,
- final Configuration conf) {
- super(nodeManager, conf);
- }
-
- /**
- * Choose datanodes called by the SCM to choose the datanode.
- *
- * @param nodesRequired - number of datanodes required.
- * @param sizeRequired - size required for the container or block.
- * @return List of Datanodes.
- * @throws SCMException SCMException
- */
- @Override
- public List<DatanodeID> chooseDatanodes(final int nodesRequired,
- final long sizeRequired) throws SCMException {
- List<DatanodeID> healthyNodes =
- super.chooseDatanodes(nodesRequired, sizeRequired);
-
- if (healthyNodes.size() == nodesRequired) {
- return healthyNodes;
- }
- return getResultSet(nodesRequired, healthyNodes);
- }
-
- /**
- * Just chose a node randomly and remove it from the set of nodes we can
- * chose from.
- *
- * @param healthyNodes - all healthy datanodes.
- * @return one randomly chosen datanode that from two randomly chosen datanode
- */
- public DatanodeID chooseNode(final List<DatanodeID> healthyNodes) {
- DatanodeID selectedNode =
- healthyNodes.get(getRand().nextInt(healthyNodes.size()));
- healthyNodes.remove(selectedNode);
- return selectedNode;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
deleted file mode 100644
index d6280df..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/algorithms/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.algorithms;
-// Various placement algorithms.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
deleted file mode 100644
index 810b8fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/ContainerStat.java
+++ /dev/null
@@ -1,166 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import java.io.IOException;
-
-import org.apache.hadoop.ozone.web.utils.JsonUtils;
-
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Preconditions;
-
-/**
- * This class represents the SCM container stat.
- */
-public class ContainerStat {
- /**
- * The maximum container size.
- */
- @JsonProperty("Size")
- private LongMetric size;
-
- /**
- * The number of bytes used by the container.
- */
- @JsonProperty("Used")
- private LongMetric used;
-
- /**
- * The number of keys in the container.
- */
- @JsonProperty("KeyCount")
- private LongMetric keyCount;
-
- /**
- * The number of bytes read from the container.
- */
- @JsonProperty("ReadBytes")
- private LongMetric readBytes;
-
- /**
- * The number of bytes write into the container.
- */
- @JsonProperty("WriteBytes")
- private LongMetric writeBytes;
-
- /**
- * The number of times the container is read.
- */
- @JsonProperty("ReadCount")
- private LongMetric readCount;
-
- /**
- * The number of times the container is written into.
- */
- @JsonProperty("WriteCount")
- private LongMetric writeCount;
-
- public ContainerStat() {
- this(0L, 0L, 0L, 0L, 0L, 0L, 0L);
- }
-
- public ContainerStat(long size, long used, long keyCount, long readBytes,
- long writeBytes, long readCount, long writeCount) {
- Preconditions.checkArgument(size >= 0,
- "Container size cannot be " + "negative.");
- Preconditions.checkArgument(used >= 0,
- "Used space cannot be " + "negative.");
- Preconditions.checkArgument(keyCount >= 0,
- "Key count cannot be " + "negative");
- Preconditions.checkArgument(readBytes >= 0,
- "Read bytes read cannot be " + "negative.");
- Preconditions.checkArgument(readBytes >= 0,
- "Write bytes cannot be " + "negative.");
- Preconditions.checkArgument(readCount >= 0,
- "Read count cannot be " + "negative.");
- Preconditions.checkArgument(writeCount >= 0,
- "Write count cannot be " + "negative");
-
- this.size = new LongMetric(size);
- this.used = new LongMetric(used);
- this.keyCount = new LongMetric(keyCount);
- this.readBytes = new LongMetric(readBytes);
- this.writeBytes = new LongMetric(writeBytes);
- this.readCount = new LongMetric(readCount);
- this.writeCount = new LongMetric(writeCount);
- }
-
- public LongMetric getSize() {
- return size;
- }
-
- public LongMetric getUsed() {
- return used;
- }
-
- public LongMetric getKeyCount() {
- return keyCount;
- }
-
- public LongMetric getReadBytes() {
- return readBytes;
- }
-
- public LongMetric getWriteBytes() {
- return writeBytes;
- }
-
- public LongMetric getReadCount() {
- return readCount;
- }
-
- public LongMetric getWriteCount() {
- return writeCount;
- }
-
- public void add(ContainerStat stat) {
- if (stat == null) {
- return;
- }
-
- this.size.add(stat.getSize().get());
- this.used.add(stat.getUsed().get());
- this.keyCount.add(stat.getKeyCount().get());
- this.readBytes.add(stat.getReadBytes().get());
- this.writeBytes.add(stat.getWriteBytes().get());
- this.readCount.add(stat.getReadCount().get());
- this.writeCount.add(stat.getWriteCount().get());
- }
-
- public void subtract(ContainerStat stat) {
- if (stat == null) {
- return;
- }
-
- this.size.subtract(stat.getSize().get());
- this.used.subtract(stat.getUsed().get());
- this.keyCount.subtract(stat.getKeyCount().get());
- this.readBytes.subtract(stat.getReadBytes().get());
- this.writeBytes.subtract(stat.getWriteBytes().get());
- this.readCount.subtract(stat.getReadCount().get());
- this.writeCount.subtract(stat.getWriteCount().get());
- }
-
- public String toJsonString() {
- try {
- return JsonUtils.toJsonString(this);
- } catch (IOException ignored) {
- return null;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
deleted file mode 100644
index cc829c2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/DatanodeMetric.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-
-/**
- * DatanodeMetric acts as the basis for all the metric that is used in
- * comparing 2 datanodes.
- */
-public interface DatanodeMetric<T, S> extends Comparable<T> {
-
- /**
- * Some syntactic sugar over Comparable interface. This makes code easier to
- * read.
- *
- * @param o - Other Object
- * @return - True if *this* object is greater than argument.
- */
- boolean isGreater(T o);
-
- /**
- * Inverse of isGreater.
- *
- * @param o - other object.
- * @return True if *this* object is Lesser than argument.
- */
- boolean isLess(T o);
-
- /**
- * Returns true if the object has same values. Because of issues with
- * equals, and loss of type information this interface supports isEqual.
- *
- * @param o object to compare.
- * @return True, if the values match.
- */
- boolean isEqual(T o);
-
- /**
- * A resourceCheck, defined by resourceNeeded.
- * For example, S could be bytes required
- * and DatanodeMetric can reply by saying it can be met or not.
- *
- * @param resourceNeeded - ResourceNeeded in its own metric.
- * @return boolean, True if this resource requirement can be met.
- */
- boolean hasResources(S resourceNeeded) throws SCMException;
-
- /**
- * Returns the metric.
- *
- * @return T, the object that represents this metric.
- */
- T get();
-
- /**
- * Sets the value of this metric.
- *
- * @param value - value of the metric.
- */
- void set(T value);
-
- /**
- * Adds a value of to the base.
- * @param value - value
- */
- void add(T value);
-
- /**
- * subtract a value.
- * @param value value
- */
- void subtract(T value);
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
deleted file mode 100644
index dbcd9f4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/LongMetric.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility;
-
-/**
- * An helper class for all metrics based on Longs.
- */
-@JsonAutoDetect(fieldVisibility = Visibility.ANY)
-public class LongMetric implements DatanodeMetric<Long, Long> {
- private Long value;
-
- /**
- * Constructs a long Metric.
- *
- * @param value Value for this metric.
- */
- public LongMetric(Long value) {
- this.value = value;
- }
-
- /**
- * Some syntactic sugar over Comparable interface. This makes code easier to
- * read.
- *
- * @param o - Other Object
- * @return - True if *this* object is greater than argument.
- */
- @Override
- public boolean isGreater(Long o) {
- return compareTo(o) > 0;
- }
-
- /**
- * Inverse of isGreater.
- *
- * @param o - other object.
- * @return True if *this* object is Lesser than argument.
- */
- @Override
- public boolean isLess(Long o) {
- return compareTo(o) < 0;
- }
-
- /**
- * Returns true if the object has same values. Because of issues with
- * equals, and loss of type information this interface supports isEqual.
- *
- * @param o object to compare.
- * @return True, if the values match.
- */
- @Override
- public boolean isEqual(Long o) {
- return compareTo(o) == 0;
- }
-
- /**
- * A resourceCheck, defined by resourceNeeded.
- * For example, S could be bytes required
- * and DatanodeMetric can reply by saying it can be met or not.
- *
- * @param resourceNeeded - ResourceNeeded in its own metric.
- * @return boolean, True if this resource requirement can be met.
- */
- @Override
- public boolean hasResources(Long resourceNeeded) {
- return isGreater(resourceNeeded);
- }
-
- /**
- * Returns the metric.
- *
- * @return T, the object that represents this metric.
- */
- @Override
- public Long get() {
- return this.value;
- }
-
- /**
- * Sets the value of this metric.
- *
- * @param setValue - value of the metric.
- */
- @Override
- public void set(Long setValue) {
- this.value = setValue;
-
- }
-
- /**
- * Adds a value of to the base.
- *
- * @param addValue - value
- */
- @Override
- public void add(Long addValue) {
- this.value += addValue;
- }
-
- /**
- * subtract a value.
- *
- * @param subValue value
- */
- @Override
- public void subtract(Long subValue) {
- this.value -= subValue;
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(Long o) {
- return Long.compare(this.value, o);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- LongMetric that = (LongMetric) o;
-
- return value != null ? value.equals(that.value) : that.value == null;
- }
-
- @Override
- public int hashCode() {
- return value != null ? value.hashCode() : 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
deleted file mode 100644
index bda61f8..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/NodeStat.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * Interface that defines Node Stats.
- */
-interface NodeStat {
- /**
- * Get capacity of the node.
- * @return capacity of the node.
- */
- LongMetric getCapacity();
-
- /**
- * Get the used space of the node.
- * @return the used space of the node.
- */
- LongMetric getScmUsed();
-
- /**
- * Get the remaining space of the node.
- * @return the remaining space of the node.
- */
- LongMetric getRemaining();
-
- /**
- * Set the total/used/remaining space.
- * @param capacity - total space.
- * @param used - used space.
- * @param remain - remaining space.
- */
- @VisibleForTesting
- void set(long capacity, long used, long remain);
-
- /**
- * Adding of the stat.
- * @param stat - stat to be added.
- * @return updated node stat.
- */
- NodeStat add(NodeStat stat);
-
- /**
- * Subtract of the stat.
- * @param stat - stat to be subtracted.
- * @return updated nodestat.
- */
- NodeStat subtract(NodeStat stat);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
deleted file mode 100644
index bdfec2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMMetrics.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.annotation.Metrics;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
-
-/**
- * This class is for maintaining StorageContainerManager statistics.
- */
-@Metrics(about="Storage Container Manager Metrics", context="dfs")
-public class SCMMetrics {
- public static final String SOURCE_NAME =
- SCMMetrics.class.getSimpleName();
-
- /**
- * Container stat metrics, the meaning of following metrics
- * can be found in {@link ContainerStat}.
- */
- @Metric private MutableGaugeLong lastContainerReportSize;
- @Metric private MutableGaugeLong lastContainerReportUsed;
- @Metric private MutableGaugeLong lastContainerReportKeyCount;
- @Metric private MutableGaugeLong lastContainerReportReadBytes;
- @Metric private MutableGaugeLong lastContainerReportWriteBytes;
- @Metric private MutableGaugeLong lastContainerReportReadCount;
- @Metric private MutableGaugeLong lastContainerReportWriteCount;
-
- @Metric private MutableCounterLong containerReportSize;
- @Metric private MutableCounterLong containerReportUsed;
- @Metric private MutableCounterLong containerReportKeyCount;
- @Metric private MutableCounterLong containerReportReadBytes;
- @Metric private MutableCounterLong containerReportWriteBytes;
- @Metric private MutableCounterLong containerReportReadCount;
- @Metric private MutableCounterLong containerReportWriteCount;
-
- public SCMMetrics() {
- }
-
- public static SCMMetrics create() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register(SOURCE_NAME, "Storage Container Manager Metrics",
- new SCMMetrics());
- }
-
- public void setLastContainerReportSize(long size) {
- this.lastContainerReportSize.set(size);
- }
-
- public void setLastContainerReportUsed(long used) {
- this.lastContainerReportUsed.set(used);
- }
-
- public void setLastContainerReportKeyCount(long keyCount) {
- this.lastContainerReportKeyCount.set(keyCount);
- }
-
- public void setLastContainerReportReadBytes(long readBytes) {
- this.lastContainerReportReadBytes.set(readBytes);
- }
-
- public void setLastContainerReportWriteBytes(long writeBytes) {
- this.lastContainerReportWriteBytes.set(writeBytes);
- }
-
- public void setLastContainerReportReadCount(long readCount) {
- this.lastContainerReportReadCount.set(readCount);
- }
-
- public void setLastContainerReportWriteCount(long writeCount) {
- this.lastContainerReportWriteCount.set(writeCount);
- }
-
- public void incrContainerReportSize(long size) {
- this.containerReportSize.incr(size);
- }
-
- public void incrContainerReportUsed(long used) {
- this.containerReportUsed.incr(used);
- }
-
- public void incrContainerReportKeyCount(long keyCount) {
- this.containerReportKeyCount.incr(keyCount);
- }
-
- public void incrContainerReportReadBytes(long readBytes) {
- this.containerReportReadBytes.incr(readBytes);
- }
-
- public void incrContainerReportWriteBytes(long writeBytes) {
- this.containerReportWriteBytes.incr(writeBytes);
- }
-
- public void incrContainerReportReadCount(long readCount) {
- this.containerReportReadCount.incr(readCount);
- }
-
- public void incrContainerReportWriteCount(long writeCount) {
- this.containerReportWriteCount.incr(writeCount);
- }
-
- public void setLastContainerStat(ContainerStat newStat) {
- this.lastContainerReportSize.set(newStat.getSize().get());
- this.lastContainerReportUsed.set(newStat.getUsed().get());
- this.lastContainerReportKeyCount.set(newStat.getKeyCount().get());
- this.lastContainerReportReadBytes.set(newStat.getReadBytes().get());
- this.lastContainerReportWriteBytes.set(newStat.getWriteBytes().get());
- this.lastContainerReportReadCount.set(newStat.getReadCount().get());
- this.lastContainerReportWriteCount.set(newStat.getWriteCount().get());
- }
-
- public void incrContainerStat(ContainerStat deltaStat) {
- this.containerReportSize.incr(deltaStat.getSize().get());
- this.containerReportUsed.incr(deltaStat.getUsed().get());
- this.containerReportKeyCount.incr(deltaStat.getKeyCount().get());
- this.containerReportReadBytes.incr(deltaStat.getReadBytes().get());
- this.containerReportWriteBytes.incr(deltaStat.getWriteBytes().get());
- this.containerReportReadCount.incr(deltaStat.getReadCount().get());
- this.containerReportWriteCount.incr(deltaStat.getWriteCount().get());
- }
-
- public void decrContainerStat(ContainerStat deltaStat) {
- this.containerReportSize.incr(-1 * deltaStat.getSize().get());
- this.containerReportUsed.incr(-1 * deltaStat.getUsed().get());
- this.containerReportKeyCount.incr(-1 * deltaStat.getKeyCount().get());
- this.containerReportReadBytes.incr(-1 * deltaStat.getReadBytes().get());
- this.containerReportWriteBytes.incr(-1 * deltaStat.getWriteBytes().get());
- this.containerReportReadCount.incr(-1 * deltaStat.getReadCount().get());
- this.containerReportWriteCount.incr(-1 * deltaStat.getWriteCount().get());
- }
-
- public void unRegister() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- ms.unregisterSource(SOURCE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
deleted file mode 100644
index ae01361..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeMetric.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-/**
- * SCM Node Metric that is used in the placement classes.
- */
-public class SCMNodeMetric implements DatanodeMetric<SCMNodeStat, Long> {
- private SCMNodeStat stat;
-
- /**
- * Constructs an SCMNode Metric.
- *
- * @param stat - SCMNodeStat.
- */
- public SCMNodeMetric(SCMNodeStat stat) {
- this.stat = stat;
- }
-
- /**
- * Set the capacity, used and remaining space on a datanode.
- *
- * @param capacity in bytes
- * @param used in bytes
- * @param remaining in bytes
- */
- @VisibleForTesting
- public SCMNodeMetric(long capacity, long used, long remaining) {
- this.stat = new SCMNodeStat();
- this.stat.set(capacity, used, remaining);
- }
-
- /**
- *
- * @param o - Other Object
- * @return - True if *this* object is greater than argument.
- */
- @Override
- public boolean isGreater(SCMNodeStat o) {
- Preconditions.checkNotNull(o, "Argument cannot be null");
-
- // if zero, replace with 1 for the division to work.
- long thisDenominator = (this.stat.getCapacity().get() == 0)
- ? 1 : this.stat.getCapacity().get();
- long otherDenominator = (o.getCapacity().get() == 0)
- ? 1 : o.getCapacity().get();
-
- float thisNodeWeight =
- stat.getScmUsed().get() / (float) thisDenominator;
-
- float oNodeWeight =
- o.getScmUsed().get() / (float) otherDenominator;
-
- if (Math.abs(thisNodeWeight - oNodeWeight) > 0.000001) {
- return thisNodeWeight > oNodeWeight;
- }
- // if these nodes are have similar weight then return the node with more
- // free space as the greater node.
- return stat.getRemaining().isGreater(o.getRemaining().get());
- }
-
- /**
- * Inverse of isGreater.
- *
- * @param o - other object.
- * @return True if *this* object is Lesser than argument.
- */
- @Override
- public boolean isLess(SCMNodeStat o) {
- Preconditions.checkNotNull(o, "Argument cannot be null");
-
- // if zero, replace with 1 for the division to work.
- long thisDenominator = (this.stat.getCapacity().get() == 0)
- ? 1 : this.stat.getCapacity().get();
- long otherDenominator = (o.getCapacity().get() == 0)
- ? 1 : o.getCapacity().get();
-
- float thisNodeWeight =
- stat.getScmUsed().get() / (float) thisDenominator;
-
- float oNodeWeight =
- o.getScmUsed().get() / (float) otherDenominator;
-
- if (Math.abs(thisNodeWeight - oNodeWeight) > 0.000001) {
- return thisNodeWeight < oNodeWeight;
- }
-
- // if these nodes are have similar weight then return the node with less
- // free space as the lesser node.
- return stat.getRemaining().isLess(o.getRemaining().get());
- }
-
- /**
- * Returns true if the object has same values. Because of issues with
- * equals, and loss of type information this interface supports isEqual.
- *
- * @param o object to compare.
- * @return True, if the values match.
- * TODO : Consider if it makes sense to add remaining to this equation.
- */
- @Override
- public boolean isEqual(SCMNodeStat o) {
- float thisNodeWeight = stat.getScmUsed().get() / (float)
- stat.getCapacity().get();
- float oNodeWeight = o.getScmUsed().get() / (float) o.getCapacity().get();
- return Math.abs(thisNodeWeight - oNodeWeight) < 0.000001;
- }
-
- /**
- * A resourceCheck, defined by resourceNeeded.
- * For example, S could be bytes required
- * and DatanodeMetric can reply by saying it can be met or not.
- *
- * @param resourceNeeded - ResourceNeeded in its own metric.
- * @return boolean, True if this resource requirement can be met.
- */
- @Override
- public boolean hasResources(Long resourceNeeded) {
- return false;
- }
-
- /**
- * Returns the metric.
- *
- * @return T, the object that represents this metric.
- */
- @Override
- public SCMNodeStat get() {
- return stat;
- }
-
- /**
- * Sets the value of this metric.
- *
- * @param value - value of the metric.
- */
- @Override
- public void set(SCMNodeStat value) {
- stat.set(value.getCapacity().get(), value.getScmUsed().get(),
- value.getRemaining().get());
- }
-
- /**
- * Adds a value of to the base.
- *
- * @param value - value
- */
- @Override
- public void add(SCMNodeStat value) {
- stat.add(value);
- }
-
- /**
- * subtract a value.
- *
- * @param value value
- */
- @Override
- public void subtract(SCMNodeStat value) {
- stat.subtract(value);
- }
-
- /**
- * Compares this object with the specified object for order. Returns a
- * negative integer, zero, or a positive integer as this object is less
- * than, equal to, or greater than the specified object.
- *
- * @param o the object to be compared.
- * @return a negative integer, zero, or a positive integer as this object is
- * less than, equal to, or greater than the specified object.
- * @throws NullPointerException if the specified object is null
- * @throws ClassCastException if the specified object's type prevents it
- * from being compared to this object.
- */
- @Override
- public int compareTo(SCMNodeStat o) {
- if (isEqual(o)) {
- return 0;
- }
- if (isGreater(o)) {
- return 1;
- } else {
- return -1;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
-
- SCMNodeMetric that = (SCMNodeMetric) o;
-
- return stat != null ? stat.equals(that.stat) : that.stat == null;
- }
-
- @Override
- public int hashCode() {
- return stat != null ? stat.hashCode() : 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
deleted file mode 100644
index 64f76a6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/SCMNodeStat.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-
-/**
- * This class represents the SCM node stat.
- */
-public class SCMNodeStat implements NodeStat {
- private LongMetric capacity;
- private LongMetric scmUsed;
- private LongMetric remaining;
-
- public SCMNodeStat() {
- this(0L, 0L, 0L);
- }
-
- public SCMNodeStat(SCMNodeStat other) {
- this(other.capacity.get(), other.scmUsed.get(), other.remaining.get());
- }
-
- public SCMNodeStat(long capacity, long used, long remaining) {
- Preconditions.checkArgument(capacity >= 0, "Capacity cannot be " +
- "negative.");
- Preconditions.checkArgument(used >= 0, "used space cannot be " +
- "negative.");
- Preconditions.checkArgument(remaining >= 0, "remaining cannot be " +
- "negative");
- this.capacity = new LongMetric(capacity);
- this.scmUsed = new LongMetric(used);
- this.remaining = new LongMetric(remaining);
- }
-
- /**
- * @return the total configured capacity of the node.
- */
- public LongMetric getCapacity() {
- return capacity;
- }
-
- /**
- * @return the total SCM used space on the node.
- */
- public LongMetric getScmUsed() {
- return scmUsed;
- }
-
- /**
- * @return the total remaining space available on the node.
- */
- public LongMetric getRemaining() {
- return remaining;
- }
-
- /**
- * Set the capacity, used and remaining space on a datanode.
- *
- * @param newCapacity in bytes
- * @param newUsed in bytes
- * @param newRemaining in bytes
- */
- @VisibleForTesting
- public void set(long newCapacity, long newUsed, long newRemaining) {
- Preconditions.checkNotNull(newCapacity, "Capacity cannot be null");
- Preconditions.checkNotNull(newUsed, "used cannot be null");
- Preconditions.checkNotNull(newRemaining, "remaining cannot be null");
-
- Preconditions.checkArgument(newCapacity >= 0, "Capacity cannot be " +
- "negative.");
- Preconditions.checkArgument(newUsed >= 0, "used space cannot be " +
- "negative.");
- Preconditions.checkArgument(newRemaining >= 0, "remaining cannot be " +
- "negative");
-
- this.capacity = new LongMetric(newCapacity);
- this.scmUsed = new LongMetric(newUsed);
- this.remaining = new LongMetric(newRemaining);
- }
-
- /**
- * Adds a new nodestat to existing values of the node.
- *
- * @param stat Nodestat.
- * @return SCMNodeStat
- */
- public SCMNodeStat add(NodeStat stat) {
- this.capacity.set(this.getCapacity().get() + stat.getCapacity().get());
- this.scmUsed.set(this.getScmUsed().get() + stat.getScmUsed().get());
- this.remaining.set(this.getRemaining().get() + stat.getRemaining().get());
- return this;
- }
-
- /**
- * Subtracts the stat values from the existing NodeStat.
- *
- * @param stat SCMNodeStat.
- * @return Modified SCMNodeStat
- */
- public SCMNodeStat subtract(NodeStat stat) {
- this.capacity.set(this.getCapacity().get() - stat.getCapacity().get());
- this.scmUsed.set(this.getScmUsed().get() - stat.getScmUsed().get());
- this.remaining.set(this.getRemaining().get() - stat.getRemaining().get());
- return this;
- }
-
- @Override
- public boolean equals(Object to) {
- if (to instanceof SCMNodeStat) {
- SCMNodeStat tempStat = (SCMNodeStat) to;
- return capacity.isEqual(tempStat.getCapacity().get()) &&
- scmUsed.isEqual(tempStat.getScmUsed().get()) &&
- remaining.isEqual(tempStat.getRemaining().get());
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- return Long.hashCode(capacity.get() ^ scmUsed.get() ^ remaining.get());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
deleted file mode 100644
index 5135095..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/metrics/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.placement.metrics;
-
-// Various metrics supported by Datanode and used by SCM in the placement
-// strategy.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
deleted file mode 100644
index 43676ef..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/placement/package-info.java
+++ /dev/null
@@ -1,19 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm.container.placement;
-// Classes related to container placement.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
deleted file mode 100644
index c063b8b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/container/replication/ContainerSupervisor.java
+++ /dev/null
@@ -1,333 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.container.replication;
-
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.scm.exceptions.SCMException;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.node.NodePoolManager;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.util.concurrent.HadoopExecutors;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.PriorityQueue;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import static com.google.common.util.concurrent.Uninterruptibles
- .sleepUninterruptibly;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS;
-import static org.apache.hadoop.scm.ScmConfigKeys
- .OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT;
-
-/**
- * This class takes a set of container reports that belong to a pool and then
- * computes the replication levels for each container.
- */
-public class ContainerSupervisor implements Closeable {
- public static final Logger LOG =
- LoggerFactory.getLogger(ContainerSupervisor.class);
-
- private final NodePoolManager poolManager;
- private final HashSet<String> poolNames;
- private final PriorityQueue<PeriodicPool> poolQueue;
- private final NodeManager nodeManager;
- private final long containerProcessingLag;
- private final AtomicBoolean runnable;
- private final ExecutorService executorService;
- private final long maxPoolWait;
- private long poolProcessCount;
- private final List<InProgressPool> inProgressPoolList;
- private final AtomicInteger threadFaultCount;
- private final int inProgressPoolMaxCount;
-
- private final ReadWriteLock inProgressPoolListLock;
-
- /**
- * Returns the number of times we have processed pools.
- * @return long
- */
- public long getPoolProcessCount() {
- return poolProcessCount;
- }
-
-
- /**
- * Constructs a class that computes Replication Levels.
- *
- * @param conf - OzoneConfiguration
- * @param nodeManager - Node Manager
- * @param poolManager - Pool Manager
- */
- public ContainerSupervisor(Configuration conf, NodeManager nodeManager,
- NodePoolManager poolManager) {
- Preconditions.checkNotNull(poolManager);
- Preconditions.checkNotNull(nodeManager);
- this.containerProcessingLag =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL,
- OZONE_SCM_CONTAINER_REPORT_PROCESSING_INTERVAL_DEFAULT,
- TimeUnit.SECONDS
- ) * 1000;
- int maxContainerReportThreads =
- conf.getInt(OZONE_SCM_MAX_CONTAINER_REPORT_THREADS,
- OZONE_SCM_MAX_CONTAINER_REPORT_THREADS_DEFAULT
- );
- this.maxPoolWait =
- conf.getTimeDuration(OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT,
- OZONE_SCM_CONTAINER_REPORTS_WAIT_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS);
- this.inProgressPoolMaxCount = conf.getInt(
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS,
- OZONE_SCM_MAX_NODEPOOL_PROCESSING_THREADS_DEFAULT);
- this.poolManager = poolManager;
- this.nodeManager = nodeManager;
- this.poolNames = new HashSet<>();
- this.poolQueue = new PriorityQueue<>();
- this.runnable = new AtomicBoolean(true);
- this.threadFaultCount = new AtomicInteger(0);
- this.executorService = HadoopExecutors.newCachedThreadPool(
- new ThreadFactoryBuilder().setDaemon(true)
- .setNameFormat("Container Reports Processing Thread - %d")
- .build(), maxContainerReportThreads);
- this.inProgressPoolList = new LinkedList<>();
- this.inProgressPoolListLock = new ReentrantReadWriteLock();
-
- initPoolProcessThread();
- }
-
- /**
- * Returns the number of pools that are under process right now.
- * @return int - Number of pools that are in process.
- */
- public int getInProgressPoolCount() {
- return inProgressPoolList.size();
- }
-
- /**
- * Exits the background thread.
- */
- public void setExit() {
- this.runnable.set(false);
- }
-
- /**
- * Adds or removes pools from names that we need to process.
- *
- * There are two different cases that we need to process.
- * The case where some pools are being added and some times we have to
- * handle cases where pools are removed.
- */
- private void refreshPools() {
- List<String> pools = this.poolManager.getNodePools();
- if (pools != null) {
-
- HashSet<String> removedPools =
- computePoolDifference(this.poolNames, new HashSet<>(pools));
-
- HashSet<String> addedPools =
- computePoolDifference(new HashSet<>(pools), this.poolNames);
- // TODO: Support remove pool API in pool manager so that this code
- // path can be tested. This never happens in the current code base.
- for (String poolName : removedPools) {
- for (PeriodicPool periodicPool : poolQueue) {
- if (periodicPool.getPoolName().compareTo(poolName) == 0) {
- poolQueue.remove(periodicPool);
- }
- }
- }
- // Remove the pool names that we have in the list.
- this.poolNames.removeAll(removedPools);
-
- for (String poolName : addedPools) {
- poolQueue.add(new PeriodicPool(poolName));
- }
-
- // Add to the pool names we are tracking.
- poolNames.addAll(addedPools);
- }
-
- }
-
- /**
- * Handle the case where pools are added.
- *
- * @param newPools - New Pools list
- * @param oldPool - oldPool List.
- */
- private HashSet<String> computePoolDifference(HashSet<String> newPools,
- Set<String> oldPool) {
- Preconditions.checkNotNull(newPools);
- Preconditions.checkNotNull(oldPool);
- HashSet<String> newSet = new HashSet<>(newPools);
- newSet.removeAll(oldPool);
- return newSet;
- }
-
- private void initPoolProcessThread() {
-
- /*
- * Task that runs to check if we need to start a pool processing job.
- * if so we create a pool reconciliation job and find out of all the
- * expected containers are on the nodes.
- */
- Runnable processPools = () -> {
- while (runnable.get()) {
- // Make sure that we don't have any new pools.
- refreshPools();
- while (inProgressPoolList.size() < inProgressPoolMaxCount) {
- PeriodicPool pool = poolQueue.poll();
- if (pool != null) {
- if (pool.getLastProcessedTime() + this.containerProcessingLag >
- Time.monotonicNow()) {
- LOG.debug("Not within the time window for processing: {}",
- pool.getPoolName());
- // we might over sleep here, not a big deal.
- sleepUninterruptibly(this.containerProcessingLag,
- TimeUnit.MILLISECONDS);
- }
- LOG.debug("Adding pool {} to container processing queue",
- pool.getPoolName());
- InProgressPool inProgressPool = new InProgressPool(maxPoolWait,
- pool, this.nodeManager, this.poolManager, this.executorService);
- inProgressPool.startReconciliation();
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.add(inProgressPool);
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- poolProcessCount++;
- } else {
- break;
- }
- }
- sleepUninterruptibly(this.maxPoolWait, TimeUnit.MILLISECONDS);
- inProgressPoolListLock.readLock().lock();
- try {
- for (InProgressPool inProgressPool : inProgressPoolList) {
- inProgressPool.finalizeReconciliation();
- poolQueue.add(inProgressPool.getPool());
- }
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- inProgressPoolListLock.writeLock().lock();
- try {
- inProgressPoolList.clear();
- } finally {
- inProgressPoolListLock.writeLock().unlock();
- }
- }
- };
-
- // We will have only one thread for pool processing.
- Thread poolProcessThread = new Thread(processPools);
- poolProcessThread.setDaemon(true);
- poolProcessThread.setName("Pool replica thread");
- poolProcessThread.setUncaughtExceptionHandler((Thread t, Throwable e) -> {
- // Let us just restart this thread after logging a critical error.
- // if this thread is not running we cannot handle commands from SCM.
- LOG.error("Critical Error : Pool replica thread encountered an " +
- "error. Thread: {} Error Count : {}", t.toString(), e,
- threadFaultCount.incrementAndGet());
- poolProcessThread.start();
- // TODO : Add a config to restrict how many times we will restart this
- // thread in a single session.
- });
- poolProcessThread.start();
- }
-
- /**
- * Adds a container report to appropriate inProgress Pool.
- * @param containerReport -- Container report for a specific container from
- * a datanode.
- */
- public void handleContainerReport(
- ContainerReportsRequestProto containerReport) {
- DatanodeID datanodeID = DatanodeID.getFromProtoBuf(
- containerReport.getDatanodeID());
- inProgressPoolListLock.readLock().lock();
- try {
- String poolName = poolManager.getNodePool(datanodeID);
- for (InProgressPool ppool : inProgressPoolList) {
- if (ppool.getPoolName().equalsIgnoreCase(poolName)) {
- ppool.handleContainerReport(containerReport);
- return;
- }
- }
- // TODO: Decide if we can do anything else with this report.
- LOG.debug("Discarding the container report for pool {}. " +
- "That pool is not currently in the pool reconciliation process." +
- " Container Name: {}", poolName, containerReport.getDatanodeID());
- } catch (SCMException e) {
- LOG.warn("Skipping processing container report from datanode {}, "
- + "cause: failed to get the corresponding node pool",
- datanodeID.toString(), e);
- } finally {
- inProgressPoolListLock.readLock().unlock();
- }
- }
-
- /**
- * Get in process pool list, used for testing.
- * @return List of InProgressPool
- */
- @VisibleForTesting
- public List<InProgressPool> getInProcessPoolList() {
- return inProgressPoolList;
- }
-
- /**
- * Shutdown the Container Replication Manager.
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- setExit();
- HadoopExecutors.shutdown(executorService, LOG, 5, TimeUnit.SECONDS);
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
new file mode 100644
index 0000000..e7df0cf
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
@@ -0,0 +1,334 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.MetricsSystem;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+
+/**
+ * This class is for maintaining the various Cblock Target statistics
+ * and publishing them through the metrics interfaces.
+ * This also registers the JMX MBean for RPC.
+ *
+ * This class maintains stats like cache hit and miss ratio
+ * as well as the latency time of read and write ops.
+ */
+public class CBlockTargetMetrics {
+ // IOPS based Metrics
+ @Metric private MutableCounterLong numReadOps;
+ @Metric private MutableCounterLong numWriteOps;
+ @Metric private MutableCounterLong numReadCacheHits;
+ @Metric private MutableCounterLong numReadCacheMiss;
+ @Metric private MutableCounterLong numDirectBlockWrites;
+
+ // Cblock internal Metrics
+ @Metric private MutableCounterLong numDirtyLogBlockRead;
+ @Metric private MutableCounterLong numBytesDirtyLogRead;
+ @Metric private MutableCounterLong numBytesDirtyLogWritten;
+ @Metric private MutableCounterLong numBlockBufferFlushCompleted;
+ @Metric private MutableCounterLong numBlockBufferFlushTriggered;
+ @Metric private MutableCounterLong numBlockBufferUpdates;
+ @Metric private MutableCounterLong numRetryLogBlockRead;
+ @Metric private MutableCounterLong numBytesRetryLogRead;
+
+ // Failure Metrics
+ @Metric private MutableCounterLong numReadLostBlocks;
+ @Metric private MutableCounterLong numFailedReadBlocks;
+ @Metric private MutableCounterLong numWriteIOExceptionRetryBlocks;
+ @Metric private MutableCounterLong numWriteGenericExceptionRetryBlocks;
+ @Metric private MutableCounterLong numFailedDirectBlockWrites;
+ @Metric private MutableCounterLong numIllegalDirtyLogFiles;
+ @Metric private MutableCounterLong numFailedDirtyLogFileDeletes;
+ @Metric private MutableCounterLong numFailedBlockBufferFlushes;
+ @Metric private MutableCounterLong numInterruptedBufferWaits;
+ @Metric private MutableCounterLong numFailedRetryLogFileWrites;
+ @Metric private MutableCounterLong numWriteMaxRetryBlocks;
+ @Metric private MutableCounterLong numFailedReleaseLevelDB;
+
+ // Latency based Metrics
+ @Metric private MutableRate dbReadLatency;
+ @Metric private MutableRate containerReadLatency;
+ @Metric private MutableRate dbWriteLatency;
+ @Metric private MutableRate containerWriteLatency;
+ @Metric private MutableRate blockBufferFlushLatency;
+ @Metric private MutableRate directBlockWriteLatency;
+
+ public CBlockTargetMetrics() {
+ }
+
+ public static CBlockTargetMetrics create() {
+ MetricsSystem ms = DefaultMetricsSystem.instance();
+ return ms.register("CBlockTargetMetrics",
+ "CBlock Target Metrics",
+ new CBlockTargetMetrics());
+ }
+
+ public void incNumReadOps() {
+ numReadOps.incr();
+ }
+
+ public void incNumWriteOps() {
+ numWriteOps.incr();
+ }
+
+ public void incNumReadCacheHits() {
+ numReadCacheHits.incr();
+ }
+
+ public void incNumReadCacheMiss() {
+ numReadCacheMiss.incr();
+ }
+
+ public void incNumReadLostBlocks() {
+ numReadLostBlocks.incr();
+ }
+
+ public void incNumDirectBlockWrites() {
+ numDirectBlockWrites.incr();
+ }
+
+ public void incNumWriteIOExceptionRetryBlocks() {
+ numWriteIOExceptionRetryBlocks.incr();
+ }
+
+ public void incNumWriteGenericExceptionRetryBlocks() {
+ numWriteGenericExceptionRetryBlocks.incr();
+ }
+
+ public void incNumFailedDirectBlockWrites() {
+ numFailedDirectBlockWrites.incr();
+ }
+
+ public void incNumFailedReadBlocks() {
+ numFailedReadBlocks.incr();
+ }
+
+ public void incNumBlockBufferFlushCompleted() {
+ numBlockBufferFlushCompleted.incr();
+ }
+
+ public void incNumBlockBufferFlushTriggered() {
+ numBlockBufferFlushTriggered.incr();
+ }
+
+ public void incNumDirtyLogBlockRead() {
+ numDirtyLogBlockRead.incr();
+ }
+
+ public void incNumBytesDirtyLogRead(int bytes) {
+ numBytesDirtyLogRead.incr(bytes);
+ }
+
+ public void incNumBlockBufferUpdates() {
+ numBlockBufferUpdates.incr();
+ }
+
+ public void incNumRetryLogBlockRead() {
+ numRetryLogBlockRead.incr();
+ }
+
+ public void incNumBytesRetryLogRead(int bytes) {
+ numBytesRetryLogRead.incr(bytes);
+ }
+
+ public void incNumBytesDirtyLogWritten(int bytes) {
+ numBytesDirtyLogWritten.incr(bytes);
+ }
+
+ public void incNumFailedBlockBufferFlushes() {
+ numFailedBlockBufferFlushes.incr();
+ }
+
+ public void incNumInterruptedBufferWaits() {
+ numInterruptedBufferWaits.incr();
+ }
+
+ public void incNumIllegalDirtyLogFiles() {
+ numIllegalDirtyLogFiles.incr();
+ }
+
+ public void incNumFailedDirtyLogFileDeletes() {
+ numFailedDirtyLogFileDeletes.incr();
+ }
+
+ public void incNumFailedRetryLogFileWrites() {
+ numFailedRetryLogFileWrites.incr();
+ }
+
+ public void incNumWriteMaxRetryBlocks() {
+ numWriteMaxRetryBlocks.incr();
+ }
+
+ public void incNumFailedReleaseLevelDB() {
+ numFailedReleaseLevelDB.incr();
+ }
+
+ public void updateDBReadLatency(long latency) {
+ dbReadLatency.add(latency);
+ }
+
+ public void updateContainerReadLatency(long latency) {
+ containerReadLatency.add(latency);
+ }
+
+ public void updateDBWriteLatency(long latency) {
+ dbWriteLatency.add(latency);
+ }
+
+ public void updateContainerWriteLatency(long latency) {
+ containerWriteLatency.add(latency);
+ }
+
+ public void updateDirectBlockWriteLatency(long latency) {
+ directBlockWriteLatency.add(latency);
+ }
+
+ public void updateBlockBufferFlushLatency(long latency) {
+ blockBufferFlushLatency.add(latency);
+ }
+
+ @VisibleForTesting
+ public long getNumReadOps() {
+ return numReadOps.value();
+ }
+
+ @VisibleForTesting
+ public long getNumWriteOps() {
+ return numWriteOps.value();
+ }
+
+ @VisibleForTesting
+ public long getNumReadCacheHits() {
+ return numReadCacheHits.value();
+ }
+
+ @VisibleForTesting
+ public long getNumReadCacheMiss() {
+ return numReadCacheMiss.value();
+ }
+
+ @VisibleForTesting
+ public long getNumReadLostBlocks() {
+ return numReadLostBlocks.value();
+ }
+
+ @VisibleForTesting
+ public long getNumDirectBlockWrites() {
+ return numDirectBlockWrites.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedDirectBlockWrites() {
+ return numFailedDirectBlockWrites.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedReadBlocks() {
+ return numFailedReadBlocks.value();
+ }
+
+ @VisibleForTesting
+ public long getNumWriteIOExceptionRetryBlocks() {
+ return numWriteIOExceptionRetryBlocks.value();
+ }
+
+ @VisibleForTesting
+ public long getNumWriteGenericExceptionRetryBlocks() {
+ return numWriteGenericExceptionRetryBlocks.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBlockBufferFlushCompleted() {
+ return numBlockBufferFlushCompleted.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBlockBufferFlushTriggered() {
+ return numBlockBufferFlushTriggered.value();
+ }
+
+ @VisibleForTesting
+ public long getNumDirtyLogBlockRead() {
+ return numDirtyLogBlockRead.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBytesDirtyLogReads() {
+ return numBytesDirtyLogRead.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBlockBufferUpdates() {
+ return numBlockBufferUpdates.value();
+ }
+
+ @VisibleForTesting
+ public long getNumRetryLogBlockRead() {
+ return numRetryLogBlockRead.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBytesRetryLogReads() {
+ return numBytesRetryLogRead.value();
+ }
+
+ @VisibleForTesting
+ public long getNumBytesDirtyLogWritten() {
+ return numBytesDirtyLogWritten.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedBlockBufferFlushes() {
+ return numFailedBlockBufferFlushes.value();
+ }
+
+ @VisibleForTesting
+ public long getNumInterruptedBufferWaits() {
+ return numInterruptedBufferWaits.value();
+ }
+
+ @VisibleForTesting
+ public long getNumIllegalDirtyLogFiles() {
+ return numIllegalDirtyLogFiles.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedDirtyLogFileDeletes() {
+ return numFailedDirtyLogFileDeletes.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedRetryLogFileWrites() {
+ return numFailedRetryLogFileWrites.value();
+ }
+
+ @VisibleForTesting
+ public long getNumWriteMaxRetryBlocks() {
+ return numWriteMaxRetryBlocks.value();
+ }
+
+ @VisibleForTesting
+ public long getNumFailedReleaseLevelDB() {
+ return numFailedReleaseLevelDB.value();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
new file mode 100644
index 0000000..75e013e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.cblock.proto.MountVolumeResponse;
+import org.apache.hadoop.cblock.util.KeyUtil;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.jscsi.target.Configuration;
+import org.jscsi.target.Target;
+import org.jscsi.target.TargetServer;
+
+import java.io.IOException;
+import java.util.HashMap;
+
+/**
+ * This class extends JSCSI target server, which is a ISCSI target that can be
+ * recognized by a remote machine with ISCSI installed.
+ */
+public final class CBlockTargetServer extends TargetServer {
+ private final OzoneConfiguration conf;
+ private final CBlockManagerHandler cBlockManagerHandler;
+ private final XceiverClientManager xceiverClientManager;
+ private final ContainerCacheFlusher containerCacheFlusher;
+ private final CBlockTargetMetrics metrics;
+
+ public CBlockTargetServer(OzoneConfiguration ozoneConfig,
+ Configuration jscsiConf,
+ CBlockManagerHandler cBlockManagerHandler,
+ CBlockTargetMetrics metrics)
+ throws IOException {
+ super(jscsiConf);
+ this.cBlockManagerHandler = cBlockManagerHandler;
+ this.xceiverClientManager = new XceiverClientManager(ozoneConfig);
+ this.conf = ozoneConfig;
+ this.containerCacheFlusher = new ContainerCacheFlusher(this.conf,
+ xceiverClientManager, metrics);
+ this.metrics = metrics;
+ LOGGER.info("Starting flusher thread.");
+ Thread flushListenerThread = new Thread(containerCacheFlusher);
+ flushListenerThread.setDaemon(true);
+ flushListenerThread.start();
+ }
+
+ public static void main(String[] args) throws Exception {
+ }
+
+ @Override
+ public boolean isValidTargetName(String checkTargetName) {
+ if (!KeyUtil.isValidVolumeKey(checkTargetName)) {
+ return false;
+ }
+ String userName = KeyUtil.getUserNameFromVolumeKey(checkTargetName);
+ String volumeName = KeyUtil.getVolumeFromVolumeKey(checkTargetName);
+ if (userName == null || volumeName == null) {
+ return false;
+ }
+ try {
+ MountVolumeResponse result =
+ cBlockManagerHandler.mountVolume(userName, volumeName);
+ if (!result.getIsValid()) {
+ LOGGER.error("Not a valid volume:" + checkTargetName);
+ return false;
+ }
+ String volumeKey = KeyUtil.getVolumeKey(result.getUserName(),
+ result.getVolumeName());
+ if (!targets.containsKey(volumeKey)) {
+ LOGGER.info("Mounting Volume. username: {} volume:{}",
+ userName, volumeName);
+ CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
+ .setUserName(userName)
+ .setVolumeName(volumeName)
+ .setVolumeSize(result.getVolumeSize())
+ .setBlockSize(result.getBlockSize())
+ .setContainerList(result.getContainerList())
+ .setClientManager(xceiverClientManager)
+ .setConf(this.conf)
+ .setFlusher(containerCacheFlusher)
+ .setCBlockTargetMetrics(metrics)
+ .build();
+ Target target = new Target(volumeKey, volumeKey, ozoneStore);
+ targets.put(volumeKey, target);
+ }
+ } catch (IOException e) {
+ LOGGER.error("Can not connect to server when validating target!"
+ + e.getMessage());
+ }
+ return targets.containsKey(checkTargetName);
+ }
+
+ @Override
+ public String[] getTargetNames() {
+ try {
+ if (cBlockManagerHandler != null) {
+ return cBlockManagerHandler.listVolumes().
+ stream().map(
+ volumeInfo -> volumeInfo.getUserName() + ":" + volumeInfo
+ .getVolumeName()).toArray(String[]::new);
+ } else {
+ return new String[0];
+ }
+ } catch (IOException e) {
+ LOGGER.error("Can't list existing volumes", e);
+ return new String[0];
+ }
+ }
+
+ @VisibleForTesting
+ public HashMap<String, Target> getTargets() {
+ return targets;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
new file mode 100644
index 0000000..292662e
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/ContainerCacheFlusher.java
@@ -0,0 +1,599 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.cblock.CBlockConfigKeys;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
+import org.apache.hadoop.cblock.jscsiHelper.cache.impl.DiskBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.iq80.leveldb.Options;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_KEEP_ALIVE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_MAX_POOL_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_THREAD_PRIORITY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT;
+
+/**
+ * Class that writes to remote containers.
+ */
+public class ContainerCacheFlusher implements Runnable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerCacheFlusher.class);
+ private final LinkedBlockingQueue<Message> messageQueue;
+ private final ThreadPoolExecutor threadPoolExecutor;
+ private final ArrayBlockingQueue<Runnable> workQueue;
+ private final ConcurrentMap<String, RefCountedDB> dbMap;
+ private final ByteBuffer blockIDBuffer;
+ private final ConcurrentMap<String, Pipeline[]> pipelineMap;
+ private final AtomicLong remoteIO;
+ private final XceiverClientManager xceiverClientManager;
+ private final CBlockTargetMetrics metrics;
+ private AtomicBoolean shutdown;
+ private final long levelDBCacheSize;
+ private final int maxRetryCount;
+ private final String tracePrefix;
+
+ private final ConcurrentMap<String, FinishCounter> finishCountMap;
+
+ /**
+ * Constructs the writers to remote queue.
+ */
+ public ContainerCacheFlusher(Configuration config,
+ XceiverClientManager xceiverClientManager,
+ CBlockTargetMetrics metrics) {
+ int queueSize = config.getInt(DFS_CBLOCK_CACHE_QUEUE_SIZE_KB,
+ DFS_CBLOCK_CACHE_QUEUE_SIZE_KB_DEFAULT) * 1024;
+ int corePoolSize = config.getInt(DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE,
+ DFS_CBLOCK_CACHE_CORE_MIN_POOL_SIZE_DEFAULT);
+ int maxPoolSize = config.getInt(DFS_CBLOCK_CACHE_MAX_POOL_SIZE,
+ DFS_CBLOCK_CACHE_MAX_POOL_SIZE_DEFAULT);
+ long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
+ DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT, TimeUnit.SECONDS);
+ int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
+ DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
+ int blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
+ DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
+ levelDBCacheSize = config.getInt(DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_KEY,
+ DFS_CBLOCK_CACHE_LEVELDB_CACHE_SIZE_MB_DEFAULT) * OzoneConsts.MB;
+
+ LOG.info("Cache: Core Pool Size: {}", corePoolSize);
+ LOG.info("Cache: Keep Alive: {}", keepAlive);
+ LOG.info("Cache: Max Pool Size: {}", maxPoolSize);
+ LOG.info("Cache: Thread Pri: {}", threadPri);
+ LOG.info("Cache: BlockBuffer Size: {}", blockBufferSize);
+
+ shutdown = new AtomicBoolean(false);
+ messageQueue = new LinkedBlockingQueue<>();
+ workQueue = new ArrayBlockingQueue<>(queueSize, true);
+
+ ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("Cache Block Writer Thread #%d")
+ .setDaemon(true)
+ .setPriority(threadPri)
+ .build();
+ threadPoolExecutor = new ThreadPoolExecutor(corePoolSize, maxPoolSize,
+ keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
+ new ThreadPoolExecutor.AbortPolicy());
+ threadPoolExecutor.prestartAllCoreThreads();
+
+ dbMap = new ConcurrentHashMap<>();
+ pipelineMap = new ConcurrentHashMap<>();
+ blockIDBuffer = ByteBuffer.allocateDirect(blockBufferSize);
+ this.xceiverClientManager = xceiverClientManager;
+ this.metrics = metrics;
+ this.remoteIO = new AtomicLong();
+
+ this.finishCountMap = new ConcurrentHashMap<>();
+ this.maxRetryCount =
+ config.getInt(CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_KEY,
+ CBlockConfigKeys.DFS_CBLOCK_CACHE_MAX_RETRY_DEFAULT);
+ this.tracePrefix = getTracePrefix();
+ }
+
+ private void checkExistingLog(String prefixFileName, File dbPath) {
+ if (!dbPath.exists()) {
+ LOG.debug("No existing dirty log found at {}", dbPath);
+ return;
+ }
+ LOG.debug("Need to check and requeue existing dirty log {}", dbPath);
+ HashMap<String, ArrayList<String>> allFiles = new HashMap<>();
+ traverse(prefixFileName, dbPath, allFiles);
+ for (Map.Entry<String, ArrayList<String>> entry : allFiles.entrySet()) {
+ String parentPath = entry.getKey();
+ for (String fileName : entry.getValue()) {
+ LOG.info("found {} {} with prefix {}",
+ parentPath, fileName, prefixFileName);
+ processDirtyBlocks(parentPath, fileName);
+ }
+ }
+ }
+
+ private void traverse(String prefixFileName, File path,
+ HashMap<String, ArrayList<String>> files) {
+ if (path.isFile()) {
+ if (path.getName().startsWith(prefixFileName)) {
+ LOG.debug("found this {} with {}", path.getParent(), path.getName());
+ if (!files.containsKey(path.getParent())) {
+ files.put(path.getParent(), new ArrayList<>());
+ }
+ files.get(path.getParent()).add(path.getName());
+ }
+ } else {
+ File[] listFiles = path.listFiles();
+ if (listFiles != null) {
+ for (File subPath : listFiles) {
+ traverse(prefixFileName, subPath, files);
+ }
+ }
+ }
+ }
+
+ /**
+ * Gets the CBlockTargetMetrics.
+ *
+ * @return CBlockTargetMetrics
+ */
+ public CBlockTargetMetrics getTargetMetrics() {
+ return metrics;
+ }
+
+ /**
+ * Gets the getXceiverClientManager.
+ *
+ * @return XceiverClientManager
+ */
+ public XceiverClientManager getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
+ /**
+ * Shutdown this instance.
+ */
+ public void shutdown() {
+ this.shutdown.set(true);
+ threadPoolExecutor.shutdown();
+ }
+
+ public long incrementRemoteIO() {
+ return remoteIO.incrementAndGet();
+ }
+
+ /**
+ * Processes a block cache file and queues those blocks for the remote I/O.
+ *
+ * @param dbPath - Location where the DB can be found.
+ * @param fileName - Block Cache File Name
+ */
+ public void processDirtyBlocks(String dbPath, String fileName) {
+ LOG.info("Adding {}/{} to queue. Queue Length: {}", dbPath, fileName,
+ messageQueue.size());
+ this.messageQueue.add(new Message(dbPath, fileName));
+ }
+
+ public Logger getLOG() {
+ return LOG;
+ }
+
+ /**
+ * Opens a DB if needed or returns a handle to an already open DB.
+ *
+ * @param dbPath -- dbPath
+ * @return the levelDB on the given path.
+ * @throws IOException
+ */
+ public synchronized LevelDBStore openDB(String dbPath)
+ throws IOException {
+ if (dbMap.containsKey(dbPath)) {
+ RefCountedDB refDB = dbMap.get(dbPath);
+ refDB.open();
+ return refDB.db;
+ } else {
+ Options options = new Options();
+ options.cacheSize(levelDBCacheSize);
+ options.createIfMissing(true);
+ LevelDBStore cacheDB = new LevelDBStore(
+ new File(getDBFileName(dbPath)), options);
+ RefCountedDB refDB = new RefCountedDB(dbPath, cacheDB);
+ dbMap.put(dbPath, refDB);
+ return cacheDB;
+ }
+ }
+
+ /**
+ * Updates the container map. This data never changes so we will update this
+ * during restarts and it should not hurt us.
+ *
+ * Once a CBlockLocalCache cache is registered, requeue dirty/retry log files
+ * for the volume
+ *
+ * @param dbPath - DbPath
+ * @param containerList - Container List.
+ */
+ public void register(String dbPath, Pipeline[] containerList) {
+ File dbFile = Paths.get(dbPath).toFile();
+ pipelineMap.put(dbPath, containerList);
+ checkExistingLog(AsyncBlockWriter.DIRTY_LOG_PREFIX, dbFile);
+ checkExistingLog(AsyncBlockWriter.RETRY_LOG_PREFIX, dbFile);
+ }
+
+ private String getDBFileName(String dbPath) {
+ return dbPath + ".db";
+ }
+
+ public LevelDBStore getCacheDB(String dbPath) throws IOException {
+ return openDB(dbPath);
+ }
+
+ public void releaseCacheDB(String dbPath) {
+ try {
+ closeDB(dbPath);
+ } catch (Exception e) {
+ metrics.incNumFailedReleaseLevelDB();
+ LOG.error("LevelDB close failed, dbPath:" + dbPath, e);
+ }
+ }
+ /**
+ * Close the DB if we don't have any outstanding references.
+ *
+ * @param dbPath - dbPath
+ * @throws IOException
+ */
+ public synchronized void closeDB(String dbPath) throws IOException {
+ if (dbMap.containsKey(dbPath)) {
+ RefCountedDB refDB = dbMap.get(dbPath);
+ int count = refDB.close();
+ if (count == 0) {
+ dbMap.remove(dbPath);
+ }
+ }
+ }
+
+ Pipeline getPipeline(String dbPath, long blockId) {
+ Pipeline[] containerList = pipelineMap.get(dbPath);
+ Preconditions.checkNotNull(containerList);
+ int containerIdx = (int) blockId % containerList.length;
+ long cBlockIndex =
+ Longs.fromByteArray(containerList[containerIdx].getData());
+ if (cBlockIndex > 0) {
+ // This catches the case when we get a wrong container in the ordering
+ // of the containers.
+ Preconditions.checkState(containerIdx % cBlockIndex == 0,
+ "The container ID computed should match with the container index " +
+ "returned from cBlock Server.");
+ }
+ return containerList[containerIdx];
+ }
+
+ public void incFinishCount(String fileName) {
+ if (!finishCountMap.containsKey(fileName)) {
+ LOG.error("No record for such file:" + fileName);
+ return;
+ }
+ finishCountMap.get(fileName).incCount();
+ if (finishCountMap.get(fileName).isFileDeleted()) {
+ finishCountMap.remove(fileName);
+ }
+ }
+
+ /**
+ * When an object implementing interface <code>Runnable</code> is used
+ * to create a thread, starting the thread causes the object's
+ * <code>run</code> method to be called in that separately executing
+ * thread.
+ * <p>
+ * The general contract of the method <code>run</code> is that it may
+ * take any action whatsoever.
+ *
+ * @see Thread#run()
+ */
+ @Override
+ public void run() {
+ while (!this.shutdown.get()) {
+ try {
+ Message message = messageQueue.take();
+ LOG.debug("Got message to process -- DB Path : {} , FileName; {}",
+ message.getDbPath(), message.getFileName());
+ String fullPath = Paths.get(message.getDbPath(),
+ message.getFileName()).toString();
+ String[] fileNameParts = message.getFileName().split("\\.");
+ Preconditions.checkState(fileNameParts.length > 1);
+ String fileType = fileNameParts[0];
+ boolean isDirtyLogFile =
+ fileType.equalsIgnoreCase(AsyncBlockWriter.DIRTY_LOG_PREFIX);
+ ReadableByteChannel fileChannel = new FileInputStream(fullPath)
+ .getChannel();
+ // TODO: We can batch and unique the IOs here. First getting the code
+ // to work, we will add those later.
+ int bytesRead = fileChannel.read(blockIDBuffer);
+ fileChannel.close();
+ LOG.debug("Read blockID log of size: {} position {} remaining {}",
+ bytesRead, blockIDBuffer.position(), blockIDBuffer.remaining());
+ // current position of in the buffer in bytes, divided by number of
+ // bytes per long (which is calculated by number of bits per long
+ // divided by number of bits per byte) gives the number of blocks
+ int blockCount = blockIDBuffer.position()/(Long.SIZE / Byte.SIZE);
+ if (isDirtyLogFile) {
+ getTargetMetrics().incNumBytesDirtyLogRead(bytesRead);
+ } else {
+ getTargetMetrics().incNumBytesRetryLogRead(bytesRead);
+ }
+ if (finishCountMap.containsKey(message.getFileName())) {
+ // In theory this should never happen. But if it happened,
+ // we need to know it...
+ getTargetMetrics().incNumIllegalDirtyLogFiles();
+ LOG.error("Adding DirtyLog file again {} current count {} new {}",
+ message.getFileName(),
+ finishCountMap.get(message.getFileName()).expectedCount,
+ blockCount);
+ }
+ finishCountMap.put(message.getFileName(),
+ new FinishCounter(blockCount, message.getDbPath(),
+ message.getFileName(), this));
+ // should be flip instead of rewind, because we also need to make sure
+ // the end position is correct.
+ blockIDBuffer.flip();
+ LOG.debug("Remaining blocks count {} and {}", blockIDBuffer.remaining(),
+ blockCount);
+ while (blockIDBuffer.remaining() >= (Long.SIZE / Byte.SIZE)) {
+ long blockID = blockIDBuffer.getLong();
+ int retryCount = 0;
+ if (isDirtyLogFile) {
+ getTargetMetrics().incNumDirtyLogBlockRead();
+ } else {
+ getTargetMetrics().incNumRetryLogBlockRead();
+ Preconditions.checkState(fileNameParts.length == 4);
+ retryCount = Integer.parseInt(fileNameParts[3]);
+ }
+ LogicalBlock block = new DiskBlock(blockID, null, false);
+ BlockWriterTask blockWriterTask = new BlockWriterTask(block, this,
+ message.getDbPath(), retryCount, message.getFileName(),
+ maxRetryCount);
+ threadPoolExecutor.submit(blockWriterTask);
+ }
+ blockIDBuffer.clear();
+ } catch (InterruptedException e) {
+ LOG.info("ContainerCacheFlusher is interrupted.", e);
+ } catch (FileNotFoundException e) {
+ LOG.error("Unable to find the dirty blocks file. This will cause " +
+ "data errors. Please stop using this volume.", e);
+ } catch (IOException e) {
+ LOG.error("Unable to read the dirty blocks file. This will cause " +
+ "data errors. Please stop using this volume.", e);
+ } catch (Exception e) {
+ LOG.error("Generic exception.", e);
+ }
+ }
+ LOG.info("Exiting flusher");
+ }
+
+ /**
+ * Tries to get the local host IP Address as trace prefix
+ * for creating trace IDs, otherwise uses a random UUID for it.
+ */
+ private static String getTracePrefix() {
+ String tmp;
+ try {
+ tmp = InetAddress.getLocalHost().getHostAddress();
+ } catch (UnknownHostException ex) {
+ tmp = UUID.randomUUID().toString();
+ LOG.error("Unable to read the host address. Using a GUID for " +
+ "hostname:{} ", tmp, ex);
+ }
+ return tmp;
+ }
+
+ /**
+ * We create a trace ID to make it easy to debug issues.
+ * A trace ID is in IPAddress:UserName:VolumeName:blockID:second format.
+ *
+ * This will get written down on the data node if we get any failures, so
+ * with this trace ID we can correlate cBlock failures across machines.
+ *
+ * @param blockID - Block ID
+ * @return trace ID
+ */
+ public String getTraceID(File dbPath, long blockID) {
+ String volumeName = dbPath.getName();
+ String userName = dbPath.getParentFile().getName();
+ // mapping to seconds to make the string smaller.
+ return tracePrefix + ":" + userName + ":" + volumeName
+ + ":" + blockID + ":" + Time.monotonicNow() / 1000;
+ }
+
+ /**
+ * Keeps a Reference counted DB that we close only when the total Reference
+ * has gone to zero.
+ */
+ private static class RefCountedDB {
+ private LevelDBStore db;
+ private AtomicInteger refcount;
+ private String dbPath;
+
+ /**
+ * RefCountedDB DB ctor.
+ *
+ * @param dbPath - DB path.
+ * @param db - LevelDBStore db
+ */
+ RefCountedDB(String dbPath, LevelDBStore db) {
+ this.db = db;
+ this.refcount = new AtomicInteger(1);
+ this.dbPath = dbPath;
+ }
+
+ /**
+ * close the DB if possible.
+ */
+ public int close() throws IOException {
+ int count = this.refcount.decrementAndGet();
+ if (count == 0) {
+ LOG.info("Closing the LevelDB. {} ", this.dbPath);
+ db.close();
+ }
+ return count;
+ }
+
+ public void open() {
+ this.refcount.incrementAndGet();
+ }
+ }
+
+ /**
+ * The message held in processing queue.
+ */
+ private static class Message {
+ private String dbPath;
+ private String fileName;
+
+ /**
+ * A message that holds the info about which path dirty blocks log and
+ * which path contains db.
+ *
+ * @param dbPath
+ * @param fileName
+ */
+ Message(String dbPath, String fileName) {
+ this.dbPath = dbPath;
+ this.fileName = fileName;
+ }
+
+ public String getDbPath() {
+ return dbPath;
+ }
+
+ public void setDbPath(String dbPath) {
+ this.dbPath = dbPath;
+ }
+
+ public String getFileName() {
+ return fileName;
+ }
+
+ public void setFileName(String fileName) {
+ this.fileName = fileName;
+ }
+ }
+
+ private static class FinishCounter {
+ private final long expectedCount;
+ private final String dbPath;
+ private final String dirtyLogPath;
+ private final AtomicLong currentCount;
+ private AtomicBoolean fileDeleted;
+ private final ContainerCacheFlusher flusher;
+
+ FinishCounter(long expectedCount, String dbPath,
+ String dirtyLogPath, ContainerCacheFlusher flusher) throws IOException {
+ this.expectedCount = expectedCount;
+ this.dbPath = dbPath;
+ this.dirtyLogPath = dirtyLogPath;
+ this.currentCount = new AtomicLong(0);
+ this.fileDeleted = new AtomicBoolean(false);
+ this.flusher = flusher;
+ }
+
+ public boolean isFileDeleted() {
+ return fileDeleted.get();
+ }
+
+ public void incCount() {
+ long count = this.currentCount.incrementAndGet();
+ if (count >= expectedCount) {
+ String filePath = String.format("%s/%s", dbPath, dirtyLogPath);
+ LOG.debug(
+ "Deleting {} with count {} {}", filePath, count, expectedCount);
+ try {
+ Path path = Paths.get(filePath);
+ Files.delete(path);
+ // the following part tries to remove the directory if it is empty
+ // but not sufficient, because the .db directory still exists....
+ // TODO how to handle the .db directory?
+ /*Path parent = path.getParent();
+ if (parent.toFile().listFiles().length == 0) {
+ Files.delete(parent);
+ }*/
+ fileDeleted.set(true);
+ } catch (Exception e) {
+ flusher.getTargetMetrics().incNumFailedDirtyLogFileDeletes();
+ LOG.error("Error deleting dirty log file:" + filePath, e);
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
new file mode 100644
index 0000000..f164f38
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/SCSITargetDaemon.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_IP;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_PORT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT;
+
+import org.apache.hadoop.cblock.CblockUtils;
+import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
+import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.hdsl.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConsts;
+import org.apache.hadoop.scm.client.ContainerOperationClient;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.jscsi.target.Configuration;
+
+import java.net.InetSocketAddress;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_CONTAINER_SIZE_GB_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_PORT_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_ADDRESS_KEY;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_DEFAULT;
+import static org.apache.hadoop.scm.ScmConfigKeys.OZONE_SCM_DATANODE_PORT_KEY;
+
+/**
+ * This class runs the target server process.
+ */
+public final class SCSITargetDaemon {
+ public static void main(String[] args) throws Exception {
+ CblockUtils.activateConfigs();
+ OzoneConfiguration ozoneConf = new OzoneConfiguration();
+
+ RPC.setProtocolEngine(ozoneConf, CBlockClientServerProtocolPB.class,
+ ProtobufRpcEngine.class);
+ long containerSizeGB = ozoneConf.getInt(DFS_CBLOCK_CONTAINER_SIZE_GB_KEY,
+ DFS_CBLOCK_CONTAINER_SIZE_GB_DEFAULT);
+ ContainerOperationClient.setContainerSizeB(
+ containerSizeGB * OzoneConsts.GB);
+ String jscsiServerAddress = ozoneConf.get(
+ DFS_CBLOCK_JSCSI_SERVER_ADDRESS_KEY,
+ DFS_CBLOCK_JSCSI_SERVER_ADDRESS_DEFAULT);
+ String cbmIPAddress = ozoneConf.get(
+ DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_KEY,
+ DFS_CBLOCK_JSCSI_CBLOCK_SERVER_ADDRESS_DEFAULT
+ );
+ int cbmPort = ozoneConf.getInt(
+ DFS_CBLOCK_JSCSI_PORT_KEY,
+ DFS_CBLOCK_JSCSI_PORT_DEFAULT
+ );
+
+ String scmAddress = ozoneConf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY,
+ OZONE_SCM_CLIENT_BIND_HOST_DEFAULT);
+ int scmClientPort = ozoneConf.getInt(OZONE_SCM_CLIENT_PORT_KEY,
+ OZONE_SCM_CLIENT_PORT_DEFAULT);
+ int scmDatanodePort = ozoneConf.getInt(OZONE_SCM_DATANODE_PORT_KEY,
+ OZONE_SCM_DATANODE_PORT_DEFAULT);
+
+ String scmClientAddress = scmAddress + ":" + scmClientPort;
+ String scmDataodeAddress = scmAddress + ":" + scmDatanodePort;
+
+ ozoneConf.set(OZONE_SCM_CLIENT_ADDRESS_KEY, scmClientAddress);
+ ozoneConf.set(OZONE_SCM_DATANODE_ADDRESS_KEY, scmDataodeAddress);
+
+ InetSocketAddress cbmAddress = new InetSocketAddress(
+ cbmIPAddress, cbmPort);
+ long version = RPC.getProtocolVersion(
+ CBlockServiceProtocolPB.class);
+ CBlockClientProtocolClientSideTranslatorPB cbmClient =
+ new CBlockClientProtocolClientSideTranslatorPB(
+ RPC.getProxy(CBlockClientServerProtocolPB.class, version,
+ cbmAddress, UserGroupInformation.getCurrentUser(), ozoneConf,
+ NetUtils.getDefaultSocketFactory(ozoneConf), 5000)
+ );
+ CBlockManagerHandler cbmHandler = new CBlockManagerHandler(cbmClient);
+
+ String advertisedAddress = ozoneConf.
+ getTrimmed(DFS_CBLOCK_ISCSI_ADVERTISED_IP, jscsiServerAddress);
+
+ int advertisedPort = ozoneConf.
+ getInt(DFS_CBLOCK_ISCSI_ADVERTISED_PORT,
+ DFS_CBLOCK_ISCSI_ADVERTISED_PORT_DEFAULT);
+
+ Configuration jscsiConfig =
+ new Configuration(jscsiServerAddress,
+ advertisedAddress,
+ advertisedPort);
+ DefaultMetricsSystem.initialize("CBlockMetrics");
+ CBlockTargetMetrics metrics = CBlockTargetMetrics.create();
+ CBlockTargetServer targetServer = new CBlockTargetServer(
+ ozoneConf, jscsiConfig, cbmHandler, metrics);
+
+ targetServer.call();
+ }
+
+ private SCSITargetDaemon() {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
new file mode 100644
index 0000000..300b2ae
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/CacheModule.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;
+
+import java.io.IOException;
+
+/**
+ * Defines the interface for cache implementations. The cache will be called
+ * by cblock storage module when it performs IO operations.
+ */
+public interface CacheModule {
+ /**
+ * check if the key is cached, if yes, returned the cached object.
+ * otherwise, load from data source. Then put it into cache.
+ *
+ * @param blockID
+ * @return the target block.
+ */
+ LogicalBlock get(long blockID) throws IOException;
+
+ /**
+ * put the value of the key into cache.
+ * @param blockID
+ * @param value
+ */
+ void put(long blockID, byte[] value) throws IOException;
+
+ void flush() throws IOException;
+
+ void start() throws IOException;
+
+ void stop() throws IOException;
+
+ void close() throws IOException;
+
+ boolean isDirtyCache();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
new file mode 100644
index 0000000..470826f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/LogicalBlock.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Logical Block is the data structure that we write to the cache,
+ * the key and data gets written to remote contianers. Rest is used for
+ * book keeping for the cache.
+ */
+public interface LogicalBlock {
+ /**
+ * Returns the data stream of this block.
+ * @return - ByteBuffer
+ */
+ ByteBuffer getData();
+
+ /**
+ * Frees the byte buffer since we don't need it any more.
+ */
+ void clearData();
+
+ /**
+ * Returns the Block ID for this Block.
+ * @return long - BlockID
+ */
+ long getBlockID();
+
+ /**
+ * Flag that tells us if this block has been persisted to container.
+ * @return whether this block is now persistent
+ */
+ boolean isPersisted();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
new file mode 100644
index 0000000..992578f
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/AsyncBlockWriter.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.commons.codec.digest.DigestUtils;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.XceiverClientSpi;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A Queue that is used to write blocks asynchronously to the container.
+ */
+public class AsyncBlockWriter {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AsyncBlockWriter.class);
+
+ /**
+ * XceiverClientManager is used to get client connections to a set of
+ * machines.
+ */
+ private final XceiverClientManager xceiverClientManager;
+
+ /**
+ * This lock is used as a signal to re-queuing thread. The requeue thread
+ * wakes up as soon as it is signaled some blocks are in the retry queue.
+ * We try really aggressively since this new block will automatically move
+ * to the end of the queue.
+ * <p>
+ * In the event a container is unavailable for a long time, we can either
+ * fail all writes or remap and let the writes succeed. The easier
+ * semantics is to fail the volume until the container is recovered by SCM.
+ */
+ private final Lock lock;
+ private final Condition notEmpty;
+ /**
+ * The cache this writer is operating against.
+ */
+ private final CBlockLocalCache parentCache;
+ private final BlockBufferManager blockBufferManager;
+ public final static String DIRTY_LOG_PREFIX = "DirtyLog";
+ public static final String RETRY_LOG_PREFIX = "RetryLog";
+ private AtomicLong localIoCount;
+
+ /**
+ * Constructs an Async Block Writer.
+ *
+ * @param config - Config
+ * @param cache - Parent Cache for this writer
+ */
+ public AsyncBlockWriter(Configuration config, CBlockLocalCache cache) {
+
+ Preconditions.checkNotNull(cache, "Cache cannot be null.");
+ Preconditions.checkNotNull(cache.getCacheDB(), "DB cannot be null.");
+ localIoCount = new AtomicLong();
+ lock = new ReentrantLock();
+ notEmpty = lock.newCondition();
+ parentCache = cache;
+ xceiverClientManager = cache.getClientManager();
+ blockBufferManager = new BlockBufferManager(config, parentCache);
+ }
+
+ public void start() throws IOException {
+ File logDir = new File(parentCache.getDbPath().toString());
+ if (!logDir.exists() && !logDir.mkdirs()) {
+ LOG.error("Unable to create the log directory, Critical error cannot " +
+ "continue. Log Dir : {}", logDir);
+ throw new IllegalStateException("Cache Directory create failed, Cannot " +
+ "continue. Log Dir: {}" + logDir);
+ }
+ blockBufferManager.start();
+ }
+
+ /**
+ * Return the log to write to.
+ *
+ * @return Logger.
+ */
+ public static Logger getLOG() {
+ return LOG;
+ }
+
+ /**
+ * Get the CacheDB.
+ *
+ * @return LevelDB Handle
+ */
+ LevelDBStore getCacheDB() {
+ return parentCache.getCacheDB();
+ }
+
+ /**
+ * Returns the client manager.
+ *
+ * @return XceiverClientManager
+ */
+ XceiverClientManager getXceiverClientManager() {
+ return xceiverClientManager;
+ }
+
+ /**
+ * Incs the localIoPacket Count that has gone into this device.
+ */
+ public long incrementLocalIO() {
+ return localIoCount.incrementAndGet();
+ }
+
+ /**
+ * Return the local io counts to this device.
+ * @return the count of io
+ */
+ public long getLocalIOCount() {
+ return localIoCount.get();
+ }
+
+ /**
+ * Writes a block to LevelDB store and queues a work item for the system to
+ * sync the block to containers.
+ *
+ * @param block - Logical Block
+ */
+ public void writeBlock(LogicalBlock block) throws IOException {
+ byte[] keybuf = Longs.toByteArray(block.getBlockID());
+ String traceID = parentCache.getTraceID(block.getBlockID());
+ if (parentCache.isShortCircuitIOEnabled()) {
+ long startTime = Time.monotonicNow();
+ getCacheDB().put(keybuf, block.getData().array());
+ incrementLocalIO();
+ long endTime = Time.monotonicNow();
+ parentCache.getTargetMetrics().updateDBWriteLatency(
+ endTime - startTime);
+ if (parentCache.isTraceEnabled()) {
+ String datahash = DigestUtils.sha256Hex(block.getData().array());
+ parentCache.getTracer().info(
+ "Task=WriterTaskDBPut,BlockID={},Time={},SHA={}",
+ block.getBlockID(), endTime - startTime, datahash);
+ }
+ block.clearData();
+ blockBufferManager.addToBlockBuffer(block.getBlockID());
+ } else {
+ Pipeline pipeline = parentCache.getPipeline(block.getBlockID());
+ String containerName = pipeline.getContainerName();
+ XceiverClientSpi client = null;
+ try {
+ long startTime = Time.monotonicNow();
+ client = parentCache.getClientManager()
+ .acquireClient(parentCache.getPipeline(block.getBlockID()));
+ ContainerProtocolCalls.writeSmallFile(client, containerName,
+ Long.toString(block.getBlockID()), block.getData().array(),
+ traceID);
+ long endTime = Time.monotonicNow();
+ if (parentCache.isTraceEnabled()) {
+ String datahash = DigestUtils.sha256Hex(block.getData().array());
+ parentCache.getTracer().info(
+ "Task=DirectWriterPut,BlockID={},Time={},SHA={}",
+ block.getBlockID(), endTime - startTime, datahash);
+ }
+ parentCache.getTargetMetrics().
+ updateDirectBlockWriteLatency(endTime - startTime);
+ parentCache.getTargetMetrics().incNumDirectBlockWrites();
+ } catch (Exception ex) {
+ parentCache.getTargetMetrics().incNumFailedDirectBlockWrites();
+ LOG.error("Direct I/O writing of block:{} traceID:{} to "
+ + "container {} failed", block.getBlockID(), traceID,
+ containerName, ex);
+ throw ex;
+ } finally {
+ if (client != null) {
+ parentCache.getClientManager().releaseClient(client);
+ }
+ block.clearData();
+ }
+ }
+ }
+
+ /**
+ * Shutdown by writing any pending I/O to dirtylog buffer.
+ */
+ public void shutdown() {
+ blockBufferManager.shutdown();
+ }
+ /**
+ * Returns tracer.
+ *
+ * @return Tracer
+ */
+ Logger getTracer() {
+ return parentCache.getTracer();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
new file mode 100644
index 0000000..c61a7a4
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferFlushTask.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import org.apache.hadoop.util.Time;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+
+/**
+ * This task is responsible for flushing the BlockIDBuffer
+ * to Dirty Log File. This Dirty Log file is used later by
+ * ContainerCacheFlusher when the data is written to container
+ */
+public class BlockBufferFlushTask implements Runnable {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BlockBufferFlushTask.class);
+ private final CBlockLocalCache parentCache;
+ private final BlockBufferManager bufferManager;
+ private final ByteBuffer blockIDBuffer;
+
+ BlockBufferFlushTask(ByteBuffer blockIDBuffer, CBlockLocalCache parentCache,
+ BlockBufferManager manager) {
+ this.parentCache = parentCache;
+ this.bufferManager = manager;
+ this.blockIDBuffer = blockIDBuffer;
+ }
+
+ /**
+ * When an object implementing interface <code>Runnable</code> is used
+ * to create a thread, starting the thread causes the object's
+ * <code>run</code> method to be called in that separately executing
+ * thread.
+ * <p>
+ * The general contract of the method <code>run</code> is that it may
+ * take any action whatsoever.
+ *
+ * @see Thread#run()
+ */
+ @Override
+ public void run() {
+ try {
+ writeBlockBufferToFile(blockIDBuffer);
+ } catch (Exception e) {
+ parentCache.getTargetMetrics().incNumFailedBlockBufferFlushes();
+ LOG.error("Unable to sync the Block map to disk with "
+ + (blockIDBuffer.position() / Long.SIZE) + "entries "
+ + "-- NOTE: This might cause a data loss or corruption", e);
+ } finally {
+ bufferManager.releaseBuffer(blockIDBuffer);
+ }
+ }
+
+ /**
+ * Write Block Buffer to file.
+ *
+ * @param buffer - ByteBuffer
+ * @throws IOException
+ */
+ private void writeBlockBufferToFile(ByteBuffer buffer)
+ throws IOException {
+ long startTime = Time.monotonicNow();
+ boolean append = false;
+
+ // If there is nothing written to blockId buffer,
+ // then skip flushing of blockId buffer
+ if (buffer.position() == 0) {
+ return;
+ }
+
+ buffer.flip();
+ String fileName =
+ String.format("%s.%s", AsyncBlockWriter.DIRTY_LOG_PREFIX,
+ Time.monotonicNow());
+ String log = Paths.get(parentCache.getDbPath().toString(), fileName)
+ .toString();
+
+ FileChannel channel = new FileOutputStream(log, append).getChannel();
+ int bytesWritten = channel.write(buffer);
+ channel.close();
+ buffer.clear();
+ parentCache.processDirtyMessage(fileName);
+ long endTime = Time.monotonicNow();
+ if (parentCache.isTraceEnabled()) {
+ parentCache.getTracer().info(
+ "Task=DirtyBlockLogWrite,Time={} bytesWritten={}",
+ endTime - startTime, bytesWritten);
+ }
+
+ parentCache.getTargetMetrics().incNumBlockBufferFlushCompleted();
+ parentCache.getTargetMetrics().incNumBytesDirtyLogWritten(bytesWritten);
+ parentCache.getTargetMetrics().
+ updateBlockBufferFlushLatency(endTime - startTime);
+ LOG.debug("Block buffer writer bytesWritten:{} Time:{}",
+ bytesWritten, endTime - startTime);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
new file mode 100644
index 0000000..5d3209c
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/BlockBufferManager.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.ThreadFactory;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_KEEP_ALIVE;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_THREAD_PRIORITY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.
+ DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT;
+
+/**
+ * This class manages the block ID buffer.
+ * Block ID Buffer keeps a list of blocks which are in leveldb cache
+ * This buffer is used later when the blocks are flushed to container
+ *
+ * Two blockIDBuffers are maintained so that write are not blocked when
+ * DirtyLog is being written. Once a blockIDBuffer is full, it will be
+ * enqueued for DirtyLog write while the other buffer accepts new write.
+ * Once the DirtyLog write is done, the buffer is returned back to the pool.
+ *
+ * There are three triggers for blockIDBuffer flush
+ * 1) BlockIDBuffer is full,
+ * 2) Time period defined for blockIDBuffer flush has elapsed.
+ * 3) Shutdown
+ */
+public class BlockBufferManager {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BlockBufferManager.class);
+
+ private enum FlushReason {
+ BUFFER_FULL,
+ SHUTDOWN,
+ TIMER
+ };
+
+ private final int blockBufferSize;
+ private final CBlockLocalCache parentCache;
+ private final ScheduledThreadPoolExecutor scheduledExecutor;
+ private final ThreadPoolExecutor threadPoolExecutor;
+ private final long intervalSeconds;
+ private final ArrayBlockingQueue<ByteBuffer> acquireQueue;
+ private final ArrayBlockingQueue<Runnable> workQueue;
+ private ByteBuffer currentBuffer;
+
+ BlockBufferManager(Configuration config, CBlockLocalCache parentCache) {
+ this.parentCache = parentCache;
+ this.scheduledExecutor = new ScheduledThreadPoolExecutor(1);
+
+ this.intervalSeconds =
+ config.getTimeDuration(DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL,
+ DFS_CBLOCK_BLOCK_BUFFER_FLUSH_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
+
+ long keepAlive = config.getTimeDuration(DFS_CBLOCK_CACHE_KEEP_ALIVE,
+ DFS_CBLOCK_CACHE_KEEP_ALIVE_DEFAULT,
+ TimeUnit.SECONDS);
+ this.workQueue = new ArrayBlockingQueue<>(2, true);
+ int threadPri = config.getInt(DFS_CBLOCK_CACHE_THREAD_PRIORITY,
+ DFS_CBLOCK_CACHE_THREAD_PRIORITY_DEFAULT);
+ ThreadFactory workerThreadFactory = new ThreadFactoryBuilder()
+ .setNameFormat("Cache Block Buffer Manager Thread #%d")
+ .setDaemon(true)
+ .setPriority(threadPri)
+ .build();
+ /*
+ * starting a thread pool with core pool size of 1 and maximum of 2 threads
+ * as there are maximum of 2 buffers which can be flushed at the same time.
+ */
+ this.threadPoolExecutor = new ThreadPoolExecutor(1, 2,
+ keepAlive, TimeUnit.SECONDS, workQueue, workerThreadFactory,
+ new ThreadPoolExecutor.AbortPolicy());
+
+ this.blockBufferSize = config.getInt(DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE,
+ DFS_CBLOCK_CACHE_BLOCK_BUFFER_SIZE_DEFAULT) * (Long.SIZE / Byte.SIZE);
+ this.acquireQueue = new ArrayBlockingQueue<>(2, true);
+
+ for (int i = 0; i < 2; i++) {
+ acquireQueue.add(ByteBuffer.allocate(blockBufferSize));
+ }
+ // get the first buffer to be used
+ this.currentBuffer = acquireQueue.remove();
+
+ LOG.info("BufferManager: Buffer Size:{} FlushIntervalSeconds:{}",
+ blockBufferSize, intervalSeconds);
+ }
+
+ // triggerBlockBufferFlush enqueues current ByteBuffer for flush and returns.
+ // This enqueue is asynchronous and hence triggerBlockBufferFlush will
+ // only block when there are no available buffers in acquireQueue
+ // Once the DirtyLog write is done, buffer is returned back to
+ // BlockBufferManager using releaseBuffer
+ private synchronized void triggerBlockBufferFlush(FlushReason reason) {
+ LOG.debug("Flush triggered because: " + reason.toString() +
+ " Num entries in buffer: " +
+ currentBuffer.position() / (Long.SIZE / Byte.SIZE) +
+ " Acquire Queue Size: " + acquireQueue.size());
+
+ parentCache.getTargetMetrics().incNumBlockBufferFlushTriggered();
+ BlockBufferFlushTask flushTask =
+ new BlockBufferFlushTask(currentBuffer, parentCache, this);
+ threadPoolExecutor.submit(flushTask);
+ try {
+ currentBuffer = acquireQueue.take();
+ } catch (InterruptedException ex) {
+ currentBuffer = null;
+ parentCache.getTargetMetrics().incNumInterruptedBufferWaits();
+ LOG.error("wait on take operation on acquire queue interrupted", ex);
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ public synchronized void addToBlockBuffer(long blockId) {
+ parentCache.getTargetMetrics().incNumBlockBufferUpdates();
+ currentBuffer.putLong(blockId);
+ // if no space left, flush this buffer
+ if (currentBuffer.remaining() == 0) {
+ triggerBlockBufferFlush(FlushReason.BUFFER_FULL);
+ }
+ }
+
+ public void releaseBuffer(ByteBuffer buffer) {
+ if (buffer.position() != 0) {
+ LOG.error("requeuing a non empty buffer with:{}",
+ "elements enqueued in the acquire queue",
+ buffer.position() / (Long.SIZE / Byte.SIZE));
+ buffer.reset();
+ }
+ // There should always be space in the queue to add an element
+ acquireQueue.add(buffer);
+ }
+
+ // Start a scheduled task to flush blockIDBuffer
+ public void start() {
+ Runnable scheduledTask = () -> triggerBlockBufferFlush(FlushReason.TIMER);
+ scheduledExecutor.scheduleWithFixedDelay(scheduledTask, intervalSeconds,
+ intervalSeconds, TimeUnit.SECONDS);
+ threadPoolExecutor.prestartAllCoreThreads();
+ }
+
+ public void shutdown() {
+ triggerBlockBufferFlush(FlushReason.SHUTDOWN);
+ scheduledExecutor.shutdown();
+ threadPoolExecutor.shutdown();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
----------------------------------------------------------------------
diff --git a/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
new file mode 100644
index 0000000..1149164
--- /dev/null
+++ b/hadoop-cblock/server/src/main/java/org/apache/hadoop/cblock/jscsiHelper/cache/impl/CBlockLocalCache.java
@@ -0,0 +1,577 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cblock.jscsiHelper.cache.impl;
+
+import com.google.common.base.Preconditions;
+import com.google.common.primitives.Longs;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.cblock.jscsiHelper.ContainerCacheFlusher;
+import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
+import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.scm.XceiverClientManager;
+import org.apache.hadoop.scm.container.common.helpers.Pipeline;
+import org.apache.hadoop.cblock.jscsiHelper.CBlockTargetMetrics;
+import org.apache.hadoop.utils.LevelDBStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.FileStore;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_DISK_CACHE_PATH_KEY;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT;
+import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
+import static org.apache.hadoop.cblock.CBlockConfigKeys
+ .DFS_CBLOCK_TRACE_IO_DEFAULT;
+
+/**
+ * A local cache used by the CBlock ISCSI server. This class is enabled or
+ * disabled via config settings.
+ */
+public class CBlockLocalCache implements CacheModule {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CBlockLocalCache.class);
+ private static final Logger TRACER =
+ LoggerFactory.getLogger("TraceIO");
+
+ private final Configuration conf;
+ /**
+ * LevelDB cache file.
+ */
+ private final LevelDBStore cacheDB;
+
+ /**
+ * AsyncBlock writer updates the cacheDB and writes the blocks async to
+ * remote containers.
+ */
+ private final AsyncBlockWriter blockWriter;
+
+ /**
+ * Sync block reader tries to read from the cache and if we get a cache
+ * miss we will fetch the block from remote location. It will asynchronously
+ * update the cacheDB.
+ */
+ private final SyncBlockReader blockReader;
+ private final String userName;
+ private final String volumeName;
+
+ /**
+ * From a block ID we are able to get the pipeline by indexing this array.
+ */
+ private final Pipeline[] containerList;
+ private final int blockSize;
+ private XceiverClientManager clientManager;
+ /**
+ * If this flag is enabled then cache traces all I/O, all reads and writes
+ * are visible in the log with sha of the block written. Makes the system
+ * slower use it only for debugging or creating trace simulations.
+ */
+ private final boolean traceEnabled;
+ private final boolean enableShortCircuitIO;
+ private final long volumeSize;
+ private long currentCacheSize;
+ private File dbPath;
+ private final ContainerCacheFlusher flusher;
+ private CBlockTargetMetrics cblockTargetMetrics;
+
+ /**
+ * Get Db Path.
+ * @return the file instance of the db.
+ */
+ public File getDbPath() {
+ return dbPath;
+ }
+
+ /**
+ * Constructor for CBlockLocalCache invoked via the builder.
+ *
+ * @param conf - Configuration
+ * @param volumeName - volume Name
+ * @param userName - user name
+ * @param containerPipelines - Pipelines that make up this contianer
+ * @param blockSize - blockSize
+ * @param flusher - flusher to flush data to container
+ * @throws IOException
+ */
+ CBlockLocalCache(
+ Configuration conf, String volumeName,
+ String userName, List<Pipeline> containerPipelines, int blockSize,
+ long volumeSize, ContainerCacheFlusher flusher) throws IOException {
+ this.conf = conf;
+ this.userName = userName;
+ this.volumeName = volumeName;
+ this.blockSize = blockSize;
+ this.flusher = flusher;
+ this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
+ DFS_CBLOCK_TRACE_IO_DEFAULT);
+ this.enableShortCircuitIO = conf.getBoolean(
+ DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO,
+ DFS_CBLOCK_ENABLE_SHORT_CIRCUIT_IO_DEFAULT);
+ dbPath = Paths.get(conf.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
+ DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT), userName, volumeName).toFile();
+
+ if (!dbPath.exists() && !dbPath.mkdirs()) {
+ LOG.error("Unable to create the cache paths. Path: {}", dbPath);
+ throw new IllegalArgumentException("Unable to create paths. Path: " +
+ dbPath);
+ }
+ cacheDB = flusher.getCacheDB(dbPath.toString());
+ this.containerList = containerPipelines.toArray(new
+ Pipeline[containerPipelines.size()]);
+ this.volumeSize = volumeSize;
+
+ blockWriter = new AsyncBlockWriter(conf, this);
+ blockReader = new SyncBlockReader(conf, this);
+ if (this.traceEnabled) {
+ getTracer().info("Task=StartingCache");
+ }
+ }
+
+ private void setClientManager(XceiverClientManager manager) {
+ this.clientManager = manager;
+ }
+
+ private void setCblockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+ this.cblockTargetMetrics = targetMetrics;
+ }
+
+ /**
+ * Returns new builder class that builds a CBlockLocalCache.
+ *
+ * @return Builder
+ */
+ public static Builder newBuilder() {
+ return new Builder();
+ }
+
+ public void processDirtyMessage(String fileName) {
+ flusher.processDirtyBlocks(dbPath.toString(), fileName);
+ }
+
+ /**
+ * Get usable disk space.
+ *
+ * @param dbPathString - Path to db
+ * @return long bytes remaining.
+ */
+ private static long getRemainingDiskSpace(String dbPathString) {
+ try {
+ URI fileUri = new URI("file:///");
+ Path dbPath = Paths.get(fileUri).resolve(dbPathString);
+ FileStore disk = Files.getFileStore(dbPath);
+ return disk.getUsableSpace();
+ } catch (URISyntaxException | IOException ex) {
+ LOG.error("Unable to get free space on for path :" + dbPathString);
+ }
+ return 0L;
+ }
+
+ /**
+ * Returns the Max current CacheSize.
+ *
+ * @return - Cache Size
+ */
+ public long getCurrentCacheSize() {
+ return currentCacheSize;
+ }
+
+ /**
+ * Sets the Maximum Cache Size.
+ *
+ * @param currentCacheSize - Max current Cache Size.
+ */
+ public void setCurrentCacheSize(long currentCacheSize) {
+ this.currentCacheSize = currentCacheSize;
+ }
+
+ /**
+ * True if block tracing is enabled.
+ *
+ * @return - bool
+ */
+ public boolean isTraceEnabled() {
+ return traceEnabled;
+ }
+
+ /**
+ * Checks if Short Circuit I/O is enabled.
+ *
+ * @return - true if it is enabled.
+ */
+ public boolean isShortCircuitIOEnabled() {
+ return enableShortCircuitIO;
+ }
+
+ /**
+ * Returns the default block size of this device.
+ *
+ * @return - int
+ */
+ public int getBlockSize() {
+ return blockSize;
+ }
+
+ /**
+ * Gets the client manager.
+ *
+ * @return XceiverClientManager
+ */
+ public XceiverClientManager getClientManager() {
+ return clientManager;
+ }
+
+ /**
+ * check if the key is cached, if yes, returned the cached object.
+ * otherwise, load from data source. Then put it into cache.
+ *
+ * @param blockID
+ * @return the block associated to the blockID
+ */
+ @Override
+ public LogicalBlock get(long blockID) throws IOException {
+ cblockTargetMetrics.incNumReadOps();
+ return blockReader.readBlock(blockID);
+ }
+
+ /**
+ * put the value of the key into cache and remote container.
+ *
+ * @param blockID - BlockID
+ * @param data - byte[]
+ */
+ @Override
+ public void put(long blockID, byte[] data) throws IOException {
+ cblockTargetMetrics.incNumWriteOps();
+ LogicalBlock block = new DiskBlock(blockID, data, false);
+ blockWriter.writeBlock(block);
+ }
+
+ @Override
+ public void flush() throws IOException {
+
+ }
+
+ @Override
+ public void start() throws IOException {
+ flusher.register(getDbPath().getPath(), containerList);
+ blockWriter.start();
+ }
+
+ @Override
+ public void stop() throws IOException {
+ }
+
+ @Override
+ public void close() throws IOException {
+ blockReader.shutdown();
+ blockWriter.shutdown();
+ this.flusher.releaseCacheDB(dbPath.toString());
+ if (this.traceEnabled) {
+ getTracer().info("Task=ShutdownCache");
+ }
+ }
+
+ /**
+ * Returns true if cache still has blocks pending to write.
+ *
+ * @return false if we have no pending blocks to write.
+ */
+ @Override
+ public boolean isDirtyCache() {
+ return false;
+ }
+
+ /**
+ * Returns the local cache DB.
+ *
+ * @return - DB
+ */
+ LevelDBStore getCacheDB() {
+ return this.cacheDB;
+ }
+
+ /**
+ * Returns the current userName.
+ *
+ * @return - UserName
+ */
+ String getUserName() {
+ return this.userName;
+ }
+
+ /**
+ * Returns the volume name.
+ *
+ * @return VolumeName.
+ */
+ String getVolumeName() {
+ return this.volumeName;
+ }
+
+ /**
+ * Returns the target metrics.
+ *
+ * @return CBlock Target Metrics.
+ */
+ CBlockTargetMetrics getTargetMetrics() {
+ return this.cblockTargetMetrics;
+ }
+
+ /**
+ * Returns the pipeline to use given a container.
+ *
+ * @param blockId - blockID
+ * @return - pipeline.
+ */
+ Pipeline getPipeline(long blockId) {
+ int containerIdx = (int) blockId % containerList.length;
+ long cBlockIndex =
+ Longs.fromByteArray(containerList[containerIdx].getData());
+ if (cBlockIndex > 0) {
+ // This catches the case when we get a wrong container in the ordering
+ // of the containers.
+ Preconditions.checkState(containerIdx % cBlockIndex == 0,
+ "The container ID computed should match with the container index " +
+ "returned from cBlock Server.");
+ }
+ return containerList[containerIdx];
+ }
+
+ String getTraceID(long blockID) {
+ return flusher.getTraceID(dbPath, blockID);
+ }
+
+ /**
+ * Returns tracer.
+ *
+ * @return - Logger
+ */
+ Logger getTracer() {
+ return TRACER;
+ }
+
+ /**
+ * Builder class for CBlocklocalCache.
+ */
+ public static class Builder {
+ private Configuration configuration;
+ private String userName;
+ private String volumeName;
+ private List<Pipeline> pipelines;
+ private XceiverClientManager clientManager;
+ private int blockSize;
+ private long volumeSize;
+ private ContainerCacheFlusher flusher;
+ private CBlockTargetMetrics metrics;
+
+ /**
+ * Ctor.
+ */
+ Builder() {
+ }
+
+ /**
+ * Computes a cache size based on the configuration and available disk
+ * space.
+ *
+ * @param configuration - Config
+ * @param volumeSize - Size of Volume
+ * @param blockSize - Size of the block
+ * @return - cache size in bytes.
+ */
+ private static long computeCacheSize(Configuration configuration,
+ long volumeSize, int blockSize) {
+ long cacheSize = 0;
+ String dbPath = configuration.get(DFS_CBLOCK_DISK_CACHE_PATH_KEY,
+ DFS_CBLOCK_DISK_CACHE_PATH_DEFAULT);
+ if (StringUtils.isBlank(dbPath)) {
+ return cacheSize;
+ }
+ long spaceRemaining = getRemainingDiskSpace(dbPath);
+ double cacheRatio = 1.0;
+
+ if (spaceRemaining < volumeSize) {
+ cacheRatio = (double)spaceRemaining / volumeSize;
+ }
+
+ // if cache is going to be at least 10% of the volume size it is worth
+ // doing, otherwise skip creating the cache.
+ if (cacheRatio >= 0.10) {
+ cacheSize = Double.doubleToLongBits(volumeSize * cacheRatio);
+ }
+ return cacheSize;
+ }
+
+ /**
+ * Sets the Config to be used by this cache.
+ *
+ * @param conf - Config
+ * @return Builder
+ */
+ public Builder setConfiguration(Configuration conf) {
+ this.configuration = conf;
+ return this;
+ }
+
+ /**
+ * Sets the user name who is the owner of this volume.
+ *
+ * @param user - name of the owner, please note this is not the current
+ * user name.
+ * @return - Builder
+ */
+ public Builder setUserName(String user) {
+ this.userName = user;
+ return this;
+ }
+
+ /**
+ * Sets the VolumeName.
+ *
+ * @param volume - Name of the volume
+ * @return Builder
+ */
+ public Builder setVolumeName(String volume) {
+ this.volumeName = volume;
+ return this;
+ }
+
+ /**
+ * Sets the Pipelines that form this volume.
+ *
+ * @param pipelineList - list of pipelines
+ * @return Builder
+ */
+ public Builder setPipelines(List<Pipeline> pipelineList) {
+ this.pipelines = pipelineList;
+ return this;
+ }
+
+ /**
+ * Sets the Client Manager that manages the communication with containers.
+ *
+ * @param xceiverClientManager - clientManager.
+ * @return - Builder
+ */
+ public Builder setClientManager(XceiverClientManager xceiverClientManager) {
+ this.clientManager = xceiverClientManager;
+ return this;
+ }
+
+ /**
+ * Sets the block size -- Typical sizes are 4KB, 8KB etc.
+ *
+ * @param size - BlockSize.
+ * @return - Builder
+ */
+ public Builder setBlockSize(int size) {
+ this.blockSize = size;
+ return this;
+ }
+
+ /**
+ * Sets the volumeSize.
+ *
+ * @param size - VolumeSize
+ * @return - Builder
+ */
+ public Builder setVolumeSize(long size) {
+ this.volumeSize = size;
+ return this;
+ }
+
+ /**
+ * Set flusher.
+ * @param containerCacheFlusher - cache Flusher
+ * @return Builder.
+ */
+ public Builder setFlusher(ContainerCacheFlusher containerCacheFlusher) {
+ this.flusher = containerCacheFlusher;
+ return this;
+ }
+
+ /**
+ * Sets the cblock Metrics.
+ *
+ * @param targetMetrics - CBlock Target Metrics
+ * @return - Builder
+ */
+ public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
+ this.metrics = targetMetrics;
+ return this;
+ }
+
+ /**
+ * Constructs a CBlockLocalCache.
+ *
+ * @return the CBlockLocalCache with the preset properties.
+ * @throws IOException
+ */
+ public CBlockLocalCache build() throws IOException {
+ Preconditions.checkNotNull(this.configuration, "A valid configuration " +
+ "is needed");
+ Preconditions.checkState(StringUtils.isNotBlank(userName), "A valid " +
+ "username is needed");
+ Preconditions.checkState(StringUtils.isNotBlank(volumeName), " A valid" +
+ " volume name is needed");
+ Preconditions.checkNotNull(this.pipelines, "Pipelines cannot be null");
+ Preconditions.checkState(this.pipelines.size() > 0, "At least one " +
+ "pipeline location is needed for a volume");
+
+ for (int x = 0; x < pipelines.size(); x++) {
+ Preconditions.checkNotNull(pipelines.get(x).getData(), "cBlock " +
+ "relies on private data on the pipeline, null data found.");
+ }
+
+ Preconditions.checkNotNull(clientManager, "Client Manager cannot be " +
+ "null");
+ Preconditions.checkState(blockSize > 0, " Block size has to be a " +
+ "number greater than 0");
+
+ Preconditions.checkState(volumeSize > 0, "Volume Size cannot be less " +
+ "than 1");
+ Preconditions.checkNotNull(this.flusher, "Flusher cannot be null.");
+
+ CBlockLocalCache cache = new CBlockLocalCache(this.configuration,
+ this.volumeName, this.userName, this.pipelines, blockSize,
+ volumeSize, flusher);
+ cache.setCblockTargetMetrics(this.metrics);
+ cache.setClientManager(this.clientManager);
+
+ // TODO : Support user configurable maximum size.
+ long cacheSize = computeCacheSize(this.configuration, this.volumeSize,
+ this.blockSize);
+ cache.setCurrentCacheSize(cacheSize);
+ return cache;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
deleted file mode 100644
index 7686df3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-/*
- * This package contains StorageContainerManager classes.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
deleted file mode 100644
index 28e5267..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineManager.java
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-
-
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-/**
- * Manage Ozone pipelines.
- */
-public abstract class PipelineManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(PipelineManager.class);
- private final List<PipelineChannel> activePipelineChannels;
- private final AtomicInteger conduitsIndex;
-
- public PipelineManager() {
- activePipelineChannels = new LinkedList<>();
- conduitsIndex = new AtomicInteger(0);
- }
-
- /**
- * This function is called by the Container Manager while allocating a new
- * container. The client specifies what kind of replication pipeline is
- * needed and based on the replication type in the request appropriate
- * Interface is invoked.
- *
- * @param containerName Name of the container
- * @param replicationFactor - Replication Factor
- * @return a Pipeline.
- */
- public synchronized final Pipeline getPipeline(String containerName,
- ReplicationFactor replicationFactor, ReplicationType replicationType)
- throws IOException {
- /**
- * In the Ozone world, we have a very simple policy.
- *
- * 1. Try to create a pipelineChannel if there are enough free nodes.
- *
- * 2. This allows all nodes to part of a pipelineChannel quickly.
- *
- * 3. if there are not enough free nodes, return conduits in a
- * round-robin fashion.
- *
- * TODO: Might have to come up with a better algorithm than this.
- * Create a new placement policy that returns conduits in round robin
- * fashion.
- */
- PipelineChannel pipelineChannel =
- allocatePipelineChannel(replicationFactor);
- if (pipelineChannel != null) {
- LOG.debug("created new pipelineChannel:{} for container:{}",
- pipelineChannel.getName(), containerName);
- activePipelineChannels.add(pipelineChannel);
- } else {
- pipelineChannel =
- findOpenPipelineChannel(replicationType, replicationFactor);
- if (pipelineChannel != null) {
- LOG.debug("re-used pipelineChannel:{} for container:{}",
- pipelineChannel.getName(), containerName);
- }
- }
- if (pipelineChannel == null) {
- LOG.error("Get pipelineChannel call failed. We are not able to find" +
- "free nodes or operational pipelineChannel.");
- return null;
- } else {
- return new Pipeline(containerName, pipelineChannel);
- }
- }
-
- protected int getReplicationCount(ReplicationFactor factor) {
- switch (factor) {
- case ONE:
- return 1;
- case THREE:
- return 3;
- default:
- throw new IllegalArgumentException("Unexpected replication count");
- }
- }
-
- public abstract PipelineChannel allocatePipelineChannel(
- ReplicationFactor replicationFactor) throws IOException;
-
- /**
- * Find a PipelineChannel that is operational.
- *
- * @return - Pipeline or null
- */
- private PipelineChannel findOpenPipelineChannel(
- ReplicationType type, ReplicationFactor factor) {
- PipelineChannel pipelineChannel = null;
- final int sentinal = -1;
- if (activePipelineChannels.size() == 0) {
- LOG.error("No Operational conduits found. Returning null.");
- return null;
- }
- int startIndex = getNextIndex();
- int nextIndex = sentinal;
- for (; startIndex != nextIndex; nextIndex = getNextIndex()) {
- // Just walk the list in a circular way.
- PipelineChannel temp =
- activePipelineChannels
- .get(nextIndex != sentinal ? nextIndex : startIndex);
- // if we find an operational pipelineChannel just return that.
- if ((temp.getLifeCycleState() == LifeCycleState.OPEN) &&
- (temp.getFactor() == factor) && (temp.getType() == type)) {
- pipelineChannel = temp;
- break;
- }
- }
- return pipelineChannel;
- }
-
- /**
- * gets the next index of the PipelineChannel to get.
- *
- * @return index in the link list to get.
- */
- private int getNextIndex() {
- return conduitsIndex.incrementAndGet() % activePipelineChannels.size();
- }
-
- /**
- * Creates a pipeline from a specified set of Nodes.
- * @param pipelineID - Name of the pipeline
- * @param datanodes - The list of datanodes that make this pipeline.
- */
- public abstract void createPipeline(String pipelineID,
- List<DatanodeID> datanodes) throws IOException;
-
- /**
- * Close the pipeline with the given clusterId.
- */
- public abstract void closePipeline(String pipelineID) throws IOException;
-
- /**
- * list members in the pipeline .
- * @return the datanode
- */
- public abstract List<DatanodeID> getMembers(String pipelineID)
- throws IOException;
-
- /**
- * Update the datanode list of the pipeline.
- */
- public abstract void updatePipeline(String pipelineID,
- List<DatanodeID> newDatanodes) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
deleted file mode 100644
index 7759f9c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/PipelineSelector.java
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.SCMContainerPlacementRandom;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.ratis.RatisManagerImpl;
-import org.apache.hadoop.ozone.scm.pipelines.standalone.StandaloneManagerImpl;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Sends the request to the right pipeline manager.
- */
-public class PipelineSelector {
- private static final Logger LOG =
- LoggerFactory.getLogger(PipelineSelector.class);
- private final ContainerPlacementPolicy placementPolicy;
- private final NodeManager nodeManager;
- private final Configuration conf;
- private final RatisManagerImpl ratisManager;
- private final StandaloneManagerImpl standaloneManager;
- private final long containerSize;
-
- /**
- * Constructs a pipeline Selector.
- *
- * @param nodeManager - node manager
- * @param conf - Ozone Config
- */
- public PipelineSelector(NodeManager nodeManager, Configuration conf) {
- this.nodeManager = nodeManager;
- this.conf = conf;
- this.placementPolicy = createContainerPlacementPolicy(nodeManager, conf);
- this.containerSize = OzoneConsts.GB * this.conf.getInt(
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_GB,
- ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT);
- this.standaloneManager =
- new StandaloneManagerImpl(this.nodeManager, placementPolicy,
- containerSize);
- this.ratisManager =
- new RatisManagerImpl(this.nodeManager, placementPolicy, containerSize,
- conf);
- }
-
- /**
- * Translates a list of nodes, ordered such that the first is the leader, into
- * a corresponding {@link Pipeline} object.
- *
- * @param nodes - list of datanodes on which we will allocate the container.
- * The first of the list will be the leader node.
- * @return pipeline corresponding to nodes
- */
- public static PipelineChannel newPipelineFromNodes(List<DatanodeID> nodes,
- LifeCycleState state, ReplicationType replicationType,
- ReplicationFactor replicationFactor, String name) {
- Preconditions.checkNotNull(nodes);
- Preconditions.checkArgument(nodes.size() > 0);
- String leaderId = nodes.get(0).getDatanodeUuid();
- PipelineChannel
- pipelineChannel = new PipelineChannel(leaderId, state, replicationType,
- replicationFactor, name);
- for (DatanodeID node : nodes) {
- pipelineChannel.addMember(node);
- }
- return pipelineChannel;
- }
-
- /**
- * Create pluggable container placement policy implementation instance.
- *
- * @param nodeManager - SCM node manager.
- * @param conf - configuration.
- * @return SCM container placement policy implementation instance.
- */
- @SuppressWarnings("unchecked")
- private static ContainerPlacementPolicy createContainerPlacementPolicy(
- final NodeManager nodeManager, final Configuration conf) {
- Class<? extends ContainerPlacementPolicy> implClass =
- (Class<? extends ContainerPlacementPolicy>) conf.getClass(
- ScmConfigKeys.OZONE_SCM_CONTAINER_PLACEMENT_IMPL_KEY,
- SCMContainerPlacementRandom.class);
-
- try {
- Constructor<? extends ContainerPlacementPolicy> ctor =
- implClass.getDeclaredConstructor(NodeManager.class,
- Configuration.class);
- return ctor.newInstance(nodeManager, conf);
- } catch (RuntimeException e) {
- throw e;
- } catch (InvocationTargetException e) {
- throw new RuntimeException(implClass.getName()
- + " could not be constructed.", e.getCause());
- } catch (Exception e) {
- LOG.error("Unhandled exception occurred, Placement policy will not be " +
- "functional.");
- throw new IllegalArgumentException("Unable to load " +
- "ContainerPlacementPolicy", e);
- }
- }
-
- /**
- * Return the pipeline manager from the replication type.
- *
- * @param replicationType - Replication Type Enum.
- * @return pipeline Manager.
- * @throws IllegalArgumentException If an pipeline type gets added
- * and this function is not modified we will throw.
- */
- private PipelineManager getPipelineManager(ReplicationType replicationType)
- throws IllegalArgumentException {
- switch (replicationType) {
- case RATIS:
- return this.ratisManager;
- case STAND_ALONE:
- return this.standaloneManager;
- case CHAINED:
- throw new IllegalArgumentException("Not implemented yet");
- default:
- throw new IllegalArgumentException("Unexpected enum found. Does not" +
- " know how to handle " + replicationType.toString());
- }
-
- }
-
- /**
- * This function is called by the Container Manager while allocating a new
- * container. The client specifies what kind of replication pipeline is needed
- * and based on the replication type in the request appropriate Interface is
- * invoked.
- */
-
- public Pipeline getReplicationPipeline(ReplicationType replicationType,
- OzoneProtos.ReplicationFactor replicationFactor, String containerName)
- throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
- Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
- LOG.debug("Getting replication pipeline for {} : Replication {}",
- containerName, replicationFactor.toString());
- return manager.
- getPipeline(containerName, replicationFactor, replicationType);
- }
-
- /**
- * Creates a pipeline from a specified set of Nodes.
- */
-
- public void createPipeline(ReplicationType replicationType, String
- pipelineID, List<DatanodeID> datanodes) throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
- Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
- LOG.debug("Creating a pipeline: {} with nodes:{}", pipelineID,
- datanodes.stream().map(DatanodeID::toString)
- .collect(Collectors.joining(",")));
- manager.createPipeline(pipelineID, datanodes);
- }
-
- /**
- * Close the pipeline with the given clusterId.
- */
-
- public void closePipeline(ReplicationType replicationType, String
- pipelineID) throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
- Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
- LOG.debug("Closing pipeline. pipelineID: {}", pipelineID);
- manager.closePipeline(pipelineID);
- }
-
- /**
- * list members in the pipeline .
- */
-
- public List<DatanodeID> getDatanodes(ReplicationType replicationType,
- String pipelineID) throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
- Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
- LOG.debug("Getting data nodes from pipeline : {}", pipelineID);
- return manager.getMembers(pipelineID);
- }
-
- /**
- * Update the datanodes in the list of the pipeline.
- */
-
- public void updateDatanodes(ReplicationType replicationType, String
- pipelineID, List<DatanodeID> newDatanodes) throws IOException {
- PipelineManager manager = getPipelineManager(replicationType);
- Preconditions.checkNotNull(manager, "Found invalid pipeline manager");
- LOG.debug("Updating pipeline: {} with new nodes:{}", pipelineID,
- newDatanodes.stream().map(DatanodeID::toString)
- .collect(Collectors.joining(",")));
- manager.updatePipeline(pipelineID, newDatanodes);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
deleted file mode 100644
index c2a3b54..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/package-info.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines;
-/**
- Ozone supports the notion of different kind of pipelines.
- That means that we can have a replication pipeline build on
- Ratis, Standalone or some other protocol. All Pipeline managers
- the entities in charge of pipelines reside in the package.
-
- Here is the high level Arch.
-
- 1. A pipeline selector class is instantiated in the Container manager class.
-
- 2. A client when creating a container -- will specify what kind of
- replication type it wants to use. We support 2 types now, Ratis and StandAlone.
-
- 3. Based on the replication type, the pipeline selector class asks the
- corresponding pipeline manager for a pipeline.
-
- 4. We have supported the ability for clients to specify a set of nodes in
- the pipeline or rely in the pipeline manager to select the datanodes if they
- are not specified.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
deleted file mode 100644
index 7903b0b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/RatisManagerImpl.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.ratis;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms
- .ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.XceiverClientRatis;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-import java.util.UUID;
-
-/**
- * Implementation of {@link PipelineManager}.
- *
- * TODO : Introduce a state machine.
- */
-public class RatisManagerImpl extends PipelineManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(RatisManagerImpl.class);
- private static final String PREFIX = "Ratis-";
- private final Configuration conf;
- private final NodeManager nodeManager;
- private final Set<DatanodeID> ratisMembers;
-
- /**
- * Constructs a Ratis Pipeline Manager.
- *
- * @param nodeManager
- */
- public RatisManagerImpl(NodeManager nodeManager,
- ContainerPlacementPolicy placementPolicy, long size, Configuration conf) {
- super();
- this.conf = conf;
- this.nodeManager = nodeManager;
- ratisMembers = new HashSet<>();
- }
-
- /**
- * Allocates a new ratis PipelineChannel from the free nodes.
- *
- * @param factor - One or Three
- * @return PipelineChannel.
- */
- public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
- List<DatanodeID> newNodesList = new LinkedList<>();
- List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
- //TODO: Add Raft State to the Nodes, so we can query and skip nodes from
- // data from datanode instead of maintaining a set.
- for (DatanodeID datanode : datanodes) {
- Preconditions.checkNotNull(datanode);
- if (!ratisMembers.contains(datanode)) {
- newNodesList.add(datanode);
- if (newNodesList.size() == count) {
- // once a datanode has been added to a pipeline, exclude it from
- // further allocations
- ratisMembers.addAll(newNodesList);
- LOG.info("Allocating a new ratis pipelineChannel of size: {}", count);
- // Start all channel names with "Ratis", easy to grep the logs.
- String conduitName = PREFIX +
- UUID.randomUUID().toString().substring(PREFIX.length());
- PipelineChannel pipelineChannel =
- PipelineSelector.newPipelineFromNodes(newNodesList,
- LifeCycleState.OPEN, ReplicationType.RATIS, factor, conduitName);
- Pipeline pipeline =
- new Pipeline("setup", pipelineChannel);
- try (XceiverClientRatis client =
- XceiverClientRatis.newXceiverClientRatis(pipeline, conf)) {
- client.createPipeline(pipeline.getPipelineName(), newNodesList);
- } catch (IOException e) {
- return null;
- }
- return pipelineChannel;
- }
- }
- }
- return null;
- }
-
- /**
- * Creates a pipeline from a specified set of Nodes.
- *
- * @param pipelineID - Name of the pipeline
- * @param datanodes - The list of datanodes that make this pipeline.
- */
- @Override
- public void createPipeline(String pipelineID, List<DatanodeID> datanodes) {
-
- }
-
- /**
- * Close the pipeline with the given clusterId.
- *
- * @param pipelineID
- */
- @Override
- public void closePipeline(String pipelineID) throws IOException {
-
- }
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeID> getMembers(String pipelineID) throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(String pipelineID, List<DatanodeID> newDatanodes)
- throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
deleted file mode 100644
index 6fe9b28..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/ratis/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.ratis;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
deleted file mode 100644
index ef37926..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/StandaloneManagerImpl.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.standalone;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationFactor;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.ReplicationType;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.LifeCycleState;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeState;
-import org.apache.hadoop.ozone.scm.container.placement.algorithms.ContainerPlacementPolicy;
-import org.apache.hadoop.ozone.scm.node.NodeManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineManager;
-import org.apache.hadoop.ozone.scm.pipelines.PipelineSelector;
-import org.apache.hadoop.scm.container.common.helpers.PipelineChannel;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.UUID;
-import java.util.Set;
-import java.util.HashSet;
-import java.util.LinkedList;
-
-/**
- * Standalone Manager Impl to prove that pluggable interface
- * works with current tests.
- */
-public class StandaloneManagerImpl extends PipelineManager {
- private static final Logger LOG =
- LoggerFactory.getLogger(StandaloneManagerImpl.class);
- private final NodeManager nodeManager;
- private final ContainerPlacementPolicy placementPolicy;
- private final long containerSize;
- private final Set<DatanodeID> standAloneMembers;
-
- /**
- * Constructor for Standalone Node Manager Impl.
- * @param nodeManager - Node Manager.
- * @param placementPolicy - Placement Policy
- * @param containerSize - Container Size.
- */
- public StandaloneManagerImpl(NodeManager nodeManager,
- ContainerPlacementPolicy placementPolicy, long containerSize) {
- super();
- this.nodeManager = nodeManager;
- this.placementPolicy = placementPolicy;
- this.containerSize = containerSize;
- this.standAloneMembers = new HashSet<>();
- }
-
-
- /**
- * Allocates a new standalone PipelineChannel from the free nodes.
- *
- * @param factor - One
- * @return PipelineChannel.
- */
- public PipelineChannel allocatePipelineChannel(ReplicationFactor factor) {
- List<DatanodeID> newNodesList = new LinkedList<>();
- List<DatanodeID> datanodes = nodeManager.getNodes(NodeState.HEALTHY);
- int count = getReplicationCount(factor);
- for (DatanodeID datanode : datanodes) {
- Preconditions.checkNotNull(datanode);
- if (!standAloneMembers.contains(datanode)) {
- newNodesList.add(datanode);
- if (newNodesList.size() == count) {
- // once a datanode has been added to a pipeline, exclude it from
- // further allocations
- standAloneMembers.addAll(newNodesList);
- LOG.info("Allocating a new standalone pipeline channel of size: {}",
- count);
- String channelName =
- "SA-" + UUID.randomUUID().toString().substring(3);
- return PipelineSelector.newPipelineFromNodes(newNodesList,
- LifeCycleState.OPEN, ReplicationType.STAND_ALONE,
- ReplicationFactor.ONE, channelName);
- }
- }
- }
- return null;
- }
-
- /**
- * Creates a pipeline from a specified set of Nodes.
- *
- * @param pipelineID - Name of the pipeline
- * @param datanodes - The list of datanodes that make this pipeline.
- */
- @Override
- public void createPipeline(String pipelineID, List<DatanodeID> datanodes) {
- //return newPipelineFromNodes(datanodes, pipelineID);
- }
-
- /**
- * Close the pipeline with the given clusterId.
- *
- * @param pipelineID
- */
- @Override
- public void closePipeline(String pipelineID) throws IOException {
-
- }
-
- /**
- * list members in the pipeline .
- *
- * @param pipelineID
- * @return the datanode
- */
- @Override
- public List<DatanodeID> getMembers(String pipelineID) throws IOException {
- return null;
- }
-
- /**
- * Update the datanode list of the pipeline.
- *
- * @param pipelineID
- * @param newDatanodes
- */
- @Override
- public void updatePipeline(String pipelineID, List<DatanodeID>
- newDatanodes) throws IOException {
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
deleted file mode 100644
index 7e6393a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/pipelines/standalone/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.pipelines.standalone;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
deleted file mode 100644
index 27fd32b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/ratis/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm.ratis;
-
-/**
- * This package contains classes related to Apache Ratis for SCM.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
deleted file mode 100644
index bec5d87..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/Freon.java
+++ /dev/null
@@ -1,1146 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.tools;
-
-import com.codahale.metrics.Histogram;
-import com.codahale.metrics.Snapshot;
-import com.codahale.metrics.UniformReservoir;
-import com.fasterxml.jackson.annotation.JsonAutoDetect;
-import com.fasterxml.jackson.annotation.PropertyAccessor;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.ObjectWriter;
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.lang.time.DurationFormatUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.*;
-import org.apache.hadoop.ozone.client.io.OzoneInputStream;
-import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
-import org.apache.hadoop.util.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.PrintStream;
-import java.text.SimpleDateFormat;
-import java.util.*;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Supplier;
-
-import static java.lang.Math.min;
-
-/**
- * Freon - A tool to populate ozone with data for testing.<br>
- * This is not a map-reduce program and this is not for benchmarking
- * Ozone write throughput.<br>
- * It supports both online and offline modes. Default mode is offline,
- * <i>-mode</i> can be used to change the mode.
- * <p>
- * In online mode, active internet connection is required,
- * common crawl data from AWS will be used.<br>
- * Default source is:<br>
- * https://commoncrawl.s3.amazonaws.com/crawl-data/
- * CC-MAIN-2017-17/warc.paths.gz<br>
- * (it contains the path to actual data segment)<br>
- * User can override this using <i>-source</i>.
- * The following values are derived from URL of Common Crawl data
- * <ul>
- * <li>Domain will be used as Volume</li>
- * <li>URL will be used as Bucket</li>
- * <li>FileName will be used as Key</li>
- * </ul></p>
- * In offline mode, the data will be random bytes and
- * size of data will be 10 KB.<br>
- * <ul>
- * <li>Default number of Volumes 10, <i>-numOfVolumes</i>
- * can be used to override</li>
- * <li>Default number of Buckets per Volume 1000, <i>-numOfBuckets</i>
- * can be used to override</li>
- * <li>Default number of Keys per Bucket 500000, <i>-numOfKeys</i>
- * can be used to override</li>
- * </ul>
- */
-public final class Freon extends Configured implements Tool {
-
- enum FreonOps {
- VOLUME_CREATE,
- BUCKET_CREATE,
- KEY_CREATE,
- KEY_WRITE
- }
-
- private static final String HELP = "help";
- private static final String MODE = "mode";
- private static final String SOURCE = "source";
- private static final String VALIDATE_WRITE = "validateWrites";
- private static final String JSON_WRITE_DIRECTORY = "jsonDir";
- private static final String NUM_OF_THREADS = "numOfThreads";
- private static final String NUM_OF_VOLUMES = "numOfVolumes";
- private static final String NUM_OF_BUCKETS = "numOfBuckets";
- private static final String NUM_OF_KEYS = "numOfKeys";
- private static final String KEY_SIZE = "keySize";
- private static final String RATIS = "ratis";
-
- private static final String MODE_DEFAULT = "offline";
- private static final String SOURCE_DEFAULT =
- "https://commoncrawl.s3.amazonaws.com/" +
- "crawl-data/CC-MAIN-2017-17/warc.paths.gz";
- private static final String NUM_OF_THREADS_DEFAULT = "10";
- private static final String NUM_OF_VOLUMES_DEFAULT = "10";
- private static final String NUM_OF_BUCKETS_DEFAULT = "1000";
- private static final String NUM_OF_KEYS_DEFAULT = "500000";
- private static final String DURATION_FORMAT = "HH:mm:ss,SSS";
-
- private static final int KEY_SIZE_DEFAULT = 10240;
- private static final int QUANTILES = 10;
-
- private static final Logger LOG =
- LoggerFactory.getLogger(Freon.class);
-
- private boolean printUsage = false;
- private boolean completed = false;
- private boolean exception = false;
-
- private String mode;
- private String source;
- private String numOfThreads;
- private String numOfVolumes;
- private String numOfBuckets;
- private String numOfKeys;
- private String jsonDir;
- private boolean useRatis;
- private ReplicationType type;
- private ReplicationFactor factor;
-
- private int threadPoolSize;
- private int keySize;
- private byte[] keyValue = null;
-
- private boolean validateWrites;
-
- private OzoneClient ozoneClient;
- private ObjectStore objectStore;
- private ExecutorService processor;
-
- private long startTime;
- private long jobStartTime;
-
- private AtomicLong volumeCreationTime;
- private AtomicLong bucketCreationTime;
- private AtomicLong keyCreationTime;
- private AtomicLong keyWriteTime;
-
- private AtomicLong totalBytesWritten;
-
- private AtomicInteger numberOfVolumesCreated;
- private AtomicInteger numberOfBucketsCreated;
- private AtomicLong numberOfKeysAdded;
-
- private Long totalWritesValidated;
- private Long writeValidationSuccessCount;
- private Long writeValidationFailureCount;
-
- private BlockingQueue<KeyValue> validationQueue;
- private ArrayList<Histogram> histograms = new ArrayList<>();
-
- @VisibleForTesting
- Freon(Configuration conf) throws IOException {
- startTime = System.nanoTime();
- jobStartTime = System.currentTimeMillis();
- volumeCreationTime = new AtomicLong();
- bucketCreationTime = new AtomicLong();
- keyCreationTime = new AtomicLong();
- keyWriteTime = new AtomicLong();
- totalBytesWritten = new AtomicLong();
- numberOfVolumesCreated = new AtomicInteger();
- numberOfBucketsCreated = new AtomicInteger();
- numberOfKeysAdded = new AtomicLong();
- ozoneClient = OzoneClientFactory.getClient(conf);
- objectStore = ozoneClient.getObjectStore();
- for (FreonOps ops : FreonOps.values()) {
- histograms.add(ops.ordinal(), new Histogram(new UniformReservoir()));
- }
- }
-
- /**
- * @param args arguments
- */
- public static void main(String[] args) throws Exception {
- Configuration conf = new OzoneConfiguration();
- int res = ToolRunner.run(conf, new Freon(conf), args);
- System.exit(res);
- }
-
- @Override
- public int run(String[] args) throws Exception {
- GenericOptionsParser parser = new GenericOptionsParser(getConf(),
- getOptions(), args);
- parseOptions(parser.getCommandLine());
- if (printUsage) {
- usage();
- return 0;
- }
-
- keyValue =
- DFSUtil.string2Bytes(RandomStringUtils.randomAscii(keySize - 36));
-
- LOG.info("Number of Threads: " + numOfThreads);
- threadPoolSize =
- min(Integer.parseInt(numOfVolumes), Integer.parseInt(numOfThreads));
- processor = Executors.newFixedThreadPool(threadPoolSize);
- addShutdownHook();
- if (mode.equals("online")) {
- LOG.info("Mode: online");
- throw new UnsupportedOperationException("Not yet implemented.");
- } else {
- LOG.info("Mode: offline");
- LOG.info("Number of Volumes: {}.", numOfVolumes);
- LOG.info("Number of Buckets per Volume: {}.", numOfBuckets);
- LOG.info("Number of Keys per Bucket: {}.", numOfKeys);
- LOG.info("Key size: {} bytes", keySize);
- for (int i = 0; i < Integer.parseInt(numOfVolumes); i++) {
- String volume = "vol-" + i + "-" +
- RandomStringUtils.randomNumeric(5);
- processor.submit(new OfflineProcessor(volume));
- }
- }
- Thread validator = null;
- if (validateWrites) {
- totalWritesValidated = 0L;
- writeValidationSuccessCount = 0L;
- writeValidationFailureCount = 0L;
-
- validationQueue =
- new ArrayBlockingQueue<>(Integer.parseInt(numOfThreads));
- validator = new Thread(new Validator());
- validator.start();
- LOG.info("Data validation is enabled.");
- }
- Thread progressbar = getProgressBarThread();
- LOG.info("Starting progress bar Thread.");
- progressbar.start();
- processor.shutdown();
- processor.awaitTermination(Integer.MAX_VALUE, TimeUnit.MILLISECONDS);
- completed = true;
- progressbar.join();
- if (validateWrites) {
- validator.join();
- }
- ozoneClient.close();
- return 0;
- }
-
- private Options getOptions() {
- Options options = new Options();
-
- OptionBuilder.withDescription("prints usage.");
- Option optHelp = OptionBuilder.create(HELP);
-
- OptionBuilder.withArgName("online | offline");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies the mode of " +
- "Freon run.");
- Option optMode = OptionBuilder.create(MODE);
-
- OptionBuilder.withArgName("source url");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies the URL of s3 " +
- "commoncrawl warc file to be used when the mode is online.");
- Option optSource = OptionBuilder.create(SOURCE);
-
- OptionBuilder.withDescription("do random validation of " +
- "data written into ozone, only subset of data is validated.");
- Option optValidateWrite = OptionBuilder.create(VALIDATE_WRITE);
-
-
- OptionBuilder.withDescription("directory where json is created");
- OptionBuilder.hasArg();
- Option optJsonDir = OptionBuilder.create(JSON_WRITE_DIRECTORY);
-
- OptionBuilder.withArgName("value");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("number of threads to be launched " +
- "for the run");
- Option optNumOfThreads = OptionBuilder.create(NUM_OF_THREADS);
-
- OptionBuilder.withArgName("value");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies number of Volumes to be " +
- "created in offline mode");
- Option optNumOfVolumes = OptionBuilder.create(NUM_OF_VOLUMES);
-
- OptionBuilder.withArgName("value");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies number of Buckets to be " +
- "created per Volume in offline mode");
- Option optNumOfBuckets = OptionBuilder.create(NUM_OF_BUCKETS);
-
- OptionBuilder.withArgName("value");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies number of Keys to be " +
- "created per Bucket in offline mode");
- Option optNumOfKeys = OptionBuilder.create(NUM_OF_KEYS);
-
- OptionBuilder.withArgName("value");
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("specifies the size of Key in bytes to be " +
- "created in offline mode");
- Option optKeySize = OptionBuilder.create(KEY_SIZE);
-
- OptionBuilder.withArgName(RATIS);
- OptionBuilder.hasArg();
- OptionBuilder.withDescription("Use Ratis as the default replication " +
- "strategy");
- Option optRatis = OptionBuilder.create(RATIS);
-
- options.addOption(optHelp);
- options.addOption(optMode);
- options.addOption(optSource);
- options.addOption(optValidateWrite);
- options.addOption(optJsonDir);
- options.addOption(optNumOfThreads);
- options.addOption(optNumOfVolumes);
- options.addOption(optNumOfBuckets);
- options.addOption(optNumOfKeys);
- options.addOption(optKeySize);
- options.addOption(optRatis);
- return options;
- }
-
- private void parseOptions(CommandLine cmdLine) {
- printUsage = cmdLine.hasOption(HELP);
-
- mode = cmdLine.getOptionValue(MODE, MODE_DEFAULT);
-
- source = cmdLine.getOptionValue(SOURCE, SOURCE_DEFAULT);
-
- numOfThreads =
- cmdLine.getOptionValue(NUM_OF_THREADS, NUM_OF_THREADS_DEFAULT);
-
- validateWrites = cmdLine.hasOption(VALIDATE_WRITE);
-
- jsonDir = cmdLine.getOptionValue(JSON_WRITE_DIRECTORY);
-
- numOfVolumes =
- cmdLine.getOptionValue(NUM_OF_VOLUMES, NUM_OF_VOLUMES_DEFAULT);
-
- numOfBuckets =
- cmdLine.getOptionValue(NUM_OF_BUCKETS, NUM_OF_BUCKETS_DEFAULT);
-
- numOfKeys = cmdLine.getOptionValue(NUM_OF_KEYS, NUM_OF_KEYS_DEFAULT);
-
- keySize = cmdLine.hasOption(KEY_SIZE) ?
- Integer.parseInt(cmdLine.getOptionValue(KEY_SIZE)) : KEY_SIZE_DEFAULT;
- if (keySize < 1024) {
- throw new IllegalArgumentException(
- "keySize can not be less than 1024 bytes");
- }
-
- useRatis = cmdLine.hasOption(RATIS);
-
- type = ReplicationType.STAND_ALONE;
- factor = ReplicationFactor.ONE;
-
- if (useRatis) {
- type = ReplicationType.RATIS;
- int replicationFactor = Integer.parseInt(cmdLine.getOptionValue(RATIS));
- switch (replicationFactor) {
- case 1:
- factor = ReplicationFactor.ONE;
- break;
- case 3:
- factor = ReplicationFactor.THREE;
- break;
- default:
- throw new IllegalArgumentException("Illegal replication factor:"
- + replicationFactor);
- }
- }
- }
-
- private void usage() {
- System.out.println("Options supported are:");
- System.out.println("-numOfThreads <value> "
- + "number of threads to be launched for the run.");
- System.out.println("-validateWrites "
- + "do random validation of data written into ozone, " +
- "only subset of data is validated.");
- System.out.println("-jsonDir "
- + "directory where json is created.");
- System.out.println("-mode [online | offline] "
- + "specifies the mode in which Freon should run.");
- System.out.println("-source <url> "
- + "specifies the URL of s3 commoncrawl warc file to " +
- "be used when the mode is online.");
- System.out.println("-numOfVolumes <value> "
- + "specifies number of Volumes to be created in offline mode");
- System.out.println("-numOfBuckets <value> "
- + "specifies number of Buckets to be created per Volume " +
- "in offline mode");
- System.out.println("-numOfKeys <value> "
- + "specifies number of Keys to be created per Bucket " +
- "in offline mode");
- System.out.println("-keySize <value> "
- + "specifies the size of Key in bytes to be created in offline mode");
- System.out.println("-help "
- + "prints usage.");
- System.out.println();
- }
-
- /**
- * Adds ShutdownHook to print statistics.
- */
- private void addShutdownHook() {
- Runtime.getRuntime().addShutdownHook(
- new Thread(() -> printStats(System.out)));
- }
-
- private Thread getProgressBarThread() {
- Supplier<Long> currentValue;
- long maxValue;
-
- if (mode.equals("online")) {
- throw new UnsupportedOperationException("Not yet implemented.");
- } else {
- currentValue = () -> numberOfKeysAdded.get();
- maxValue = Long.parseLong(numOfVolumes) *
- Long.parseLong(numOfBuckets) *
- Long.parseLong(numOfKeys);
- }
- Thread progressBarThread = new Thread(
- new ProgressBar(System.out, currentValue, maxValue));
- progressBarThread.setName("ProgressBar");
- return progressBarThread;
- }
-
- /**
- * Prints stats of {@link Freon} run to the PrintStream.
- *
- * @param out PrintStream
- */
- private void printStats(PrintStream out) {
- long endTime = System.nanoTime() - startTime;
- String execTime = DurationFormatUtils
- .formatDuration(TimeUnit.NANOSECONDS.toMillis(endTime),
- DURATION_FORMAT);
-
- long volumeTime = TimeUnit.NANOSECONDS.toMillis(volumeCreationTime.get())
- / threadPoolSize;
- String prettyAverageVolumeTime =
- DurationFormatUtils.formatDuration(volumeTime, DURATION_FORMAT);
-
- long bucketTime = TimeUnit.NANOSECONDS.toMillis(bucketCreationTime.get())
- / threadPoolSize;
- String prettyAverageBucketTime =
- DurationFormatUtils.formatDuration(bucketTime, DURATION_FORMAT);
-
- long averageKeyCreationTime =
- TimeUnit.NANOSECONDS.toMillis(keyCreationTime.get())
- / threadPoolSize;
- String prettyAverageKeyCreationTime = DurationFormatUtils
- .formatDuration(averageKeyCreationTime, DURATION_FORMAT);
-
- long averageKeyWriteTime =
- TimeUnit.NANOSECONDS.toMillis(keyWriteTime.get()) / threadPoolSize;
- String prettyAverageKeyWriteTime = DurationFormatUtils
- .formatDuration(averageKeyWriteTime, DURATION_FORMAT);
-
- out.println();
- out.println("***************************************************");
- out.println("Status: " + (exception ? "Failed" : "Success"));
- out.println("Git Base Revision: " + VersionInfo.getRevision());
- out.println("Number of Volumes created: " + numberOfVolumesCreated);
- out.println("Number of Buckets created: " + numberOfBucketsCreated);
- out.println("Number of Keys added: " + numberOfKeysAdded);
- out.println("Ratis replication factor: " + factor.name());
- out.println("Ratis replication type: " + type.name());
- out.println(
- "Average Time spent in volume creation: " + prettyAverageVolumeTime);
- out.println(
- "Average Time spent in bucket creation: " + prettyAverageBucketTime);
- out.println(
- "Average Time spent in key creation: " + prettyAverageKeyCreationTime);
- out.println(
- "Average Time spent in key write: " + prettyAverageKeyWriteTime);
- out.println("Total bytes written: " + totalBytesWritten);
- if (validateWrites) {
- out.println("Total number of writes validated: " +
- totalWritesValidated);
- out.println("Writes validated: " +
- (100.0 * totalWritesValidated / numberOfKeysAdded.get())
- + " %");
- out.println("Successful validation: " +
- writeValidationSuccessCount);
- out.println("Unsuccessful validation: " +
- writeValidationFailureCount);
- }
- out.println("Total Execution time: " + execTime);
- out.println("***************************************************");
-
- if (jsonDir != null) {
-
- String[][] quantileTime =
- new String[FreonOps.values().length][QUANTILES + 1];
- String[] deviations = new String[FreonOps.values().length];
- String[] means = new String[FreonOps.values().length];
- for (FreonOps ops : FreonOps.values()) {
- Snapshot snapshot = histograms.get(ops.ordinal()).getSnapshot();
- for (int i = 0; i <= QUANTILES; i++) {
- quantileTime[ops.ordinal()][i] = DurationFormatUtils.formatDuration(
- TimeUnit.NANOSECONDS
- .toMillis((long) snapshot.getValue((1.0 / QUANTILES) * i)),
- DURATION_FORMAT);
- }
- deviations[ops.ordinal()] = DurationFormatUtils.formatDuration(
- TimeUnit.NANOSECONDS.toMillis((long) snapshot.getStdDev()),
- DURATION_FORMAT);
- means[ops.ordinal()] = DurationFormatUtils.formatDuration(
- TimeUnit.NANOSECONDS.toMillis((long) snapshot.getMean()),
- DURATION_FORMAT);
- }
-
- FreonJobInfo jobInfo = new FreonJobInfo().setExecTime(execTime)
- .setGitBaseRevision(VersionInfo.getRevision())
- .setMeanVolumeCreateTime(means[FreonOps.VOLUME_CREATE.ordinal()])
- .setDeviationVolumeCreateTime(
- deviations[FreonOps.VOLUME_CREATE.ordinal()])
- .setTenQuantileVolumeCreateTime(
- quantileTime[FreonOps.VOLUME_CREATE.ordinal()])
- .setMeanBucketCreateTime(means[FreonOps.BUCKET_CREATE.ordinal()])
- .setDeviationBucketCreateTime(
- deviations[FreonOps.BUCKET_CREATE.ordinal()])
- .setTenQuantileBucketCreateTime(
- quantileTime[FreonOps.BUCKET_CREATE.ordinal()])
- .setMeanKeyCreateTime(means[FreonOps.KEY_CREATE.ordinal()])
- .setDeviationKeyCreateTime(deviations[FreonOps.KEY_CREATE.ordinal()])
- .setTenQuantileKeyCreateTime(
- quantileTime[FreonOps.KEY_CREATE.ordinal()])
- .setMeanKeyWriteTime(means[FreonOps.KEY_WRITE.ordinal()])
- .setDeviationKeyWriteTime(deviations[FreonOps.KEY_WRITE.ordinal()])
- .setTenQuantileKeyWriteTime(
- quantileTime[FreonOps.KEY_WRITE.ordinal()]);
- String jsonName =
- new SimpleDateFormat("yyyyMMddHHmmss").format(Time.now()) + ".json";
- String jsonPath = jsonDir + "/" + jsonName;
- FileOutputStream os = null;
- try {
- os = new FileOutputStream(jsonPath);
- ObjectMapper mapper = new ObjectMapper();
- mapper.setVisibility(PropertyAccessor.FIELD,
- JsonAutoDetect.Visibility.ANY);
- ObjectWriter writer = mapper.writerWithDefaultPrettyPrinter();
- writer.writeValue(os, jobInfo);
- } catch (FileNotFoundException e) {
- out.println("Json File could not be created for the path: " + jsonPath);
- out.println(e);
- } catch (IOException e) {
- out.println("Json object could not be created");
- out.println(e);
- } finally {
- try {
- if (os != null) {
- os.close();
- }
- } catch (IOException e) {
- LOG.warn("Could not close the output stream for json", e);
- }
- }
- }
- }
-
- /**
- * Returns the number of volumes created.
- * @return volume count.
- */
- @VisibleForTesting
- int getNumberOfVolumesCreated() {
- return numberOfVolumesCreated.get();
- }
-
- /**
- * Returns the number of buckets created.
- * @return bucket count.
- */
- @VisibleForTesting
- int getNumberOfBucketsCreated() {
- return numberOfBucketsCreated.get();
- }
-
- /**
- * Returns the number of keys added.
- * @return keys count.
- */
- @VisibleForTesting
- long getNumberOfKeysAdded() {
- return numberOfKeysAdded.get();
- }
-
- /**
- * Returns true if random validation of write is enabled.
- * @return validateWrites
- */
- @VisibleForTesting
- boolean getValidateWrites() {
- return validateWrites;
- }
-
- /**
- * Returns the number of keys validated.
- * @return validated key count.
- */
- @VisibleForTesting
- long getTotalKeysValidated() {
- return totalWritesValidated;
- }
-
- /**
- * Returns the number of successful validation.
- * @return successful validation count.
- */
- @VisibleForTesting
- long getSuccessfulValidationCount() {
- return writeValidationSuccessCount;
- }
-
- /**
- * Returns the number of unsuccessful validation.
- * @return unsuccessful validation count.
- */
- @VisibleForTesting
- long getUnsuccessfulValidationCount() {
- return writeValidationFailureCount;
- }
-
- /**
- * Returns the length of the common key value initialized.
- * @return key value length initialized.
- */
- @VisibleForTesting
- long getKeyValueLength(){
- return keyValue.length;
- }
-
- /**
- * Wrapper to hold ozone key-value pair.
- */
- private static class KeyValue {
-
- /**
- * Bucket name associated with the key-value.
- */
- private OzoneBucket bucket;
- /**
- * Key name associated with the key-value.
- */
- private String key;
- /**
- * Value associated with the key-value.
- */
- private byte[] value;
-
- /**
- * Constructs a new ozone key-value pair.
- *
- * @param key key part
- * @param value value part
- */
- KeyValue(OzoneBucket bucket, String key, byte[] value) {
- this.bucket = bucket;
- this.key = key;
- this.value = value;
- }
- }
-
- private class OfflineProcessor implements Runnable {
-
- private int totalBuckets;
- private int totalKeys;
- private String volumeName;
-
- OfflineProcessor(String volumeName) {
- this.totalBuckets = Integer.parseInt(numOfBuckets);
- this.totalKeys = Integer.parseInt(numOfKeys);
- this.volumeName = volumeName;
- }
-
- @Override
- public void run() {
- LOG.trace("Creating volume: {}", volumeName);
- long start = System.nanoTime();
- OzoneVolume volume;
- try {
- objectStore.createVolume(volumeName);
- long volumeCreationDuration = System.nanoTime() - start;
- volumeCreationTime.getAndAdd(volumeCreationDuration);
- histograms.get(FreonOps.VOLUME_CREATE.ordinal())
- .update(volumeCreationDuration);
- numberOfVolumesCreated.getAndIncrement();
- volume = objectStore.getVolume(volumeName);
- } catch (IOException e) {
- exception = true;
- LOG.error("Could not create volume", e);
- return;
- }
-
- Long threadKeyWriteTime = 0L;
- for (int j = 0; j < totalBuckets; j++) {
- String bucketName = "bucket-" + j + "-" +
- RandomStringUtils.randomNumeric(5);
- try {
- LOG.trace("Creating bucket: {} in volume: {}",
- bucketName, volume.getName());
- start = System.nanoTime();
- volume.createBucket(bucketName);
- long bucketCreationDuration = System.nanoTime() - start;
- histograms.get(FreonOps.BUCKET_CREATE.ordinal())
- .update(bucketCreationDuration);
- bucketCreationTime.getAndAdd(bucketCreationDuration);
- numberOfBucketsCreated.getAndIncrement();
- OzoneBucket bucket = volume.getBucket(bucketName);
- for (int k = 0; k < totalKeys; k++) {
- String key = "key-" + k + "-" +
- RandomStringUtils.randomNumeric(5);
- byte[] randomValue =
- DFSUtil.string2Bytes(UUID.randomUUID().toString());
- try {
- LOG.trace("Adding key: {} in bucket: {} of volume: {}",
- key, bucket, volume);
- long keyCreateStart = System.nanoTime();
- OzoneOutputStream os =
- bucket.createKey(key, keySize, type, factor);
- long keyCreationDuration = System.nanoTime() - keyCreateStart;
- histograms.get(FreonOps.KEY_CREATE.ordinal())
- .update(keyCreationDuration);
- keyCreationTime.getAndAdd(keyCreationDuration);
- long keyWriteStart = System.nanoTime();
- os.write(keyValue);
- os.write(randomValue);
- os.close();
- long keyWriteDuration = System.nanoTime() - keyWriteStart;
- threadKeyWriteTime += keyWriteDuration;
- histograms.get(FreonOps.KEY_WRITE.ordinal())
- .update(keyWriteDuration);
- totalBytesWritten.getAndAdd(keySize);
- numberOfKeysAdded.getAndIncrement();
- if (validateWrites) {
- byte[] value = ArrayUtils.addAll(keyValue, randomValue);
- boolean validate = validationQueue.offer(
- new KeyValue(bucket, key, value));
- if (validate) {
- LOG.trace("Key {}, is queued for validation.", key);
- }
- }
- } catch (Exception e) {
- exception = true;
- LOG.error("Exception while adding key: {} in bucket: {}" +
- " of volume: {}.", key, bucket, volume, e);
- }
- }
- } catch (Exception e) {
- exception = true;
- LOG.error("Exception while creating bucket: {}" +
- " in volume: {}.", bucketName, volume, e);
- }
- }
-
- keyWriteTime.getAndAdd(threadKeyWriteTime);
- }
-
- }
-
- private final class FreonJobInfo {
-
- private String status;
- private String gitBaseRevision;
- private String jobStartTime;
- private String numOfVolumes;
- private String numOfBuckets;
- private String numOfKeys;
- private String numOfThreads;
- private String mode;
- private String dataWritten;
- private String execTime;
- private String replicationFactor;
- private String replicationType;
-
- private int keySize;
-
- private String totalThroughputPerSecond;
-
- private String meanVolumeCreateTime;
- private String deviationVolumeCreateTime;
- private String[] tenQuantileVolumeCreateTime;
-
- private String meanBucketCreateTime;
- private String deviationBucketCreateTime;
- private String[] tenQuantileBucketCreateTime;
-
- private String meanKeyCreateTime;
- private String deviationKeyCreateTime;
- private String[] tenQuantileKeyCreateTime;
-
- private String meanKeyWriteTime;
- private String deviationKeyWriteTime;
- private String[] tenQuantileKeyWriteTime;
-
- private FreonJobInfo() {
- this.status = exception ? "Failed" : "Success";
- this.numOfVolumes = Freon.this.numOfVolumes;
- this.numOfBuckets = Freon.this.numOfBuckets;
- this.numOfKeys = Freon.this.numOfKeys;
- this.numOfThreads = Freon.this.numOfThreads;
- this.keySize = Freon.this.keySize;
- this.mode = Freon.this.mode;
- this.jobStartTime = Time.formatTime(Freon.this.jobStartTime);
- this.replicationFactor = Freon.this.factor.name();
- this.replicationType = Freon.this.type.name();
-
- long totalBytes =
- Long.parseLong(numOfVolumes) * Long.parseLong(numOfBuckets) * Long
- .parseLong(numOfKeys) * keySize;
- this.dataWritten = getInStorageUnits((double) totalBytes);
- this.totalThroughputPerSecond = getInStorageUnits(
- (totalBytes * 1.0) / TimeUnit.NANOSECONDS
- .toSeconds(Freon.this.keyWriteTime.get() / threadPoolSize));
- }
-
- private String getInStorageUnits(Double value) {
- double size;
- OzoneQuota.Units unit;
- if ((long) (value / OzoneConsts.TB) != 0) {
- size = value / OzoneConsts.TB;
- unit = OzoneQuota.Units.TB;
- } else if ((long) (value / OzoneConsts.GB) != 0) {
- size = value / OzoneConsts.GB;
- unit = OzoneQuota.Units.GB;
- } else if ((long) (value / OzoneConsts.MB) != 0) {
- size = value / OzoneConsts.MB;
- unit = OzoneQuota.Units.MB;
- } else if ((long) (value / OzoneConsts.KB) != 0) {
- size = value / OzoneConsts.KB;
- unit = OzoneQuota.Units.KB;
- } else {
- size = value;
- unit = OzoneQuota.Units.BYTES;
- }
- return size + " " + unit;
- }
-
- public FreonJobInfo setGitBaseRevision(String gitBaseRevisionVal) {
- gitBaseRevision = gitBaseRevisionVal;
- return this;
- }
-
- public FreonJobInfo setExecTime(String execTimeVal) {
- execTime = execTimeVal;
- return this;
- }
-
- public FreonJobInfo setMeanKeyWriteTime(String deviationKeyWriteTimeVal) {
- this.meanKeyWriteTime = deviationKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setDeviationKeyWriteTime(
- String deviationKeyWriteTimeVal) {
- this.deviationKeyWriteTime = deviationKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setTenQuantileKeyWriteTime(
- String[] tenQuantileKeyWriteTimeVal) {
- this.tenQuantileKeyWriteTime = tenQuantileKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setMeanKeyCreateTime(String deviationKeyWriteTimeVal) {
- this.meanKeyCreateTime = deviationKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setDeviationKeyCreateTime(
- String deviationKeyCreateTimeVal) {
- this.deviationKeyCreateTime = deviationKeyCreateTimeVal;
- return this;
- }
-
- public FreonJobInfo setTenQuantileKeyCreateTime(
- String[] tenQuantileKeyCreateTimeVal) {
- this.tenQuantileKeyCreateTime = tenQuantileKeyCreateTimeVal;
- return this;
- }
-
- public FreonJobInfo setMeanBucketCreateTime(
- String deviationKeyWriteTimeVal) {
- this.meanBucketCreateTime = deviationKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setDeviationBucketCreateTime(
- String deviationBucketCreateTimeVal) {
- this.deviationBucketCreateTime = deviationBucketCreateTimeVal;
- return this;
- }
-
- public FreonJobInfo setTenQuantileBucketCreateTime(
- String[] tenQuantileBucketCreateTimeVal) {
- this.tenQuantileBucketCreateTime = tenQuantileBucketCreateTimeVal;
- return this;
- }
-
- public FreonJobInfo setMeanVolumeCreateTime(
- String deviationKeyWriteTimeVal) {
- this.meanVolumeCreateTime = deviationKeyWriteTimeVal;
- return this;
- }
-
- public FreonJobInfo setDeviationVolumeCreateTime(
- String deviationVolumeCreateTimeVal) {
- this.deviationVolumeCreateTime = deviationVolumeCreateTimeVal;
- return this;
- }
-
- public FreonJobInfo setTenQuantileVolumeCreateTime(
- String[] tenQuantileVolumeCreateTimeVal) {
- this.tenQuantileVolumeCreateTime = tenQuantileVolumeCreateTimeVal;
- return this;
- }
-
- public String getJobStartTime() {
- return jobStartTime;
- }
-
- public String getNumOfVolumes() {
- return numOfVolumes;
- }
-
- public String getNumOfBuckets() {
- return numOfBuckets;
- }
-
- public String getNumOfKeys() {
- return numOfKeys;
- }
-
- public String getNumOfThreads() {
- return numOfThreads;
- }
-
- public String getMode() {
- return mode;
- }
-
- public String getExecTime() {
- return execTime;
- }
-
- public String getReplicationFactor() {
- return replicationFactor;
- }
-
- public String getReplicationType() {
- return replicationType;
- }
-
- public String getStatus() {
- return status;
- }
-
- public int getKeySize() {
- return keySize;
- }
-
- public String getGitBaseRevision() {
- return gitBaseRevision;
- }
-
- public String getDataWritten() {
- return dataWritten;
- }
-
- public String getTotalThroughputPerSecond() {
- return totalThroughputPerSecond;
- }
-
- public String getMeanVolumeCreateTime() {
- return meanVolumeCreateTime;
- }
-
- public String getDeviationVolumeCreateTime() {
- return deviationVolumeCreateTime;
- }
-
- public String[] getTenQuantileVolumeCreateTime() {
- return tenQuantileVolumeCreateTime;
- }
-
- public String getMeanBucketCreateTime() {
- return meanBucketCreateTime;
- }
-
- public String getDeviationBucketCreateTime() {
- return deviationBucketCreateTime;
- }
-
- public String[] getTenQuantileBucketCreateTime() {
- return tenQuantileBucketCreateTime;
- }
-
- public String getMeanKeyCreateTime() {
- return meanKeyCreateTime;
- }
-
- public String getDeviationKeyCreateTime() {
- return deviationKeyCreateTime;
- }
-
- public String[] getTenQuantileKeyCreateTime() {
- return tenQuantileKeyCreateTime;
- }
-
- public String getMeanKeyWriteTime() {
- return meanKeyWriteTime;
- }
-
- public String getDeviationKeyWriteTime() {
- return deviationKeyWriteTime;
- }
-
- public String[] getTenQuantileKeyWriteTime() {
- return tenQuantileKeyWriteTime;
- }
- }
-
- private class ProgressBar implements Runnable {
-
- private static final long REFRESH_INTERVAL = 1000L;
-
- private PrintStream stream;
- private Supplier<Long> currentValue;
- private long maxValue;
-
- ProgressBar(PrintStream stream, Supplier<Long> currentValue,
- long maxValue) {
- this.stream = stream;
- this.currentValue = currentValue;
- this.maxValue = maxValue;
- }
-
- @Override
- public void run() {
- try {
- stream.println();
- long value;
- while ((value = currentValue.get()) < maxValue) {
- print(value);
- if (completed) {
- break;
- }
- Thread.sleep(REFRESH_INTERVAL);
- }
- if (exception) {
- stream.println();
- stream.println("Incomplete termination, " +
- "check log for exception.");
- } else {
- print(maxValue);
- }
- stream.println();
- } catch (InterruptedException e) {
- }
- }
-
- /**
- * Given current value prints the progress bar.
- *
- * @param value
- */
- private void print(long value) {
- stream.print('\r');
- double percent = 100.0 * value / maxValue;
- StringBuilder sb = new StringBuilder();
- sb.append(" " + String.format("%.2f", percent) + "% |");
-
- for (int i = 0; i <= percent; i++) {
- sb.append('█');
- }
- for (int j = 0; j < 100 - percent; j++) {
- sb.append(' ');
- }
- sb.append("| ");
- sb.append(value + "/" + maxValue);
- long timeInSec = TimeUnit.SECONDS.convert(
- System.nanoTime() - startTime, TimeUnit.NANOSECONDS);
- String timeToPrint = String.format("%d:%02d:%02d", timeInSec / 3600,
- (timeInSec % 3600) / 60, timeInSec % 60);
- sb.append(" Time: " + timeToPrint);
- stream.print(sb);
- }
- }
-
- /**
- * Validates the write done in ozone cluster.
- */
- private class Validator implements Runnable {
-
- @Override
- public void run() {
- while (!completed) {
- try {
- KeyValue kv = validationQueue.poll(5, TimeUnit.SECONDS);
- if (kv != null) {
-
- OzoneInputStream is = kv.bucket.readKey(kv.key);
- byte[] value = new byte[kv.value.length];
- int length = is.read(value);
- totalWritesValidated++;
- if (length == kv.value.length && Arrays.equals(value, kv.value)) {
- writeValidationSuccessCount++;
- } else {
- writeValidationFailureCount++;
- LOG.warn("Data validation error for key {}/{}/{}",
- kv.bucket.getVolumeName(), kv.bucket, kv.key);
- LOG.warn("Expected: {}, Actual: {}",
- DFSUtil.bytes2String(kv.value),
- DFSUtil.bytes2String(value));
- }
- }
- } catch (IOException | InterruptedException ex) {
- LOG.error("Exception while validating write: " + ex.getMessage());
- }
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
deleted file mode 100644
index 4d74f48..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/tools/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.tools;
-
-/**
- This package contains class used for testing and benchmarking ozone cluster.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
deleted file mode 100644
index 7dd9a33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/ObjectStoreApplication.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import org.apache.hadoop.ozone.client.rest.OzoneExceptionMapper;
-import org.apache.hadoop.ozone.web.handlers.BucketHandler;
-import org.apache.hadoop.ozone.web.handlers.KeyHandler;
-import org.apache.hadoop.ozone.web.handlers.ServiceFilter;
-import org.apache.hadoop.ozone.web.handlers.VolumeHandler;
-import org.apache.hadoop.ozone.web.messages.LengthInputStreamMessageBodyWriter;
-import org.apache.hadoop.ozone.web.messages.StringMessageBodyWriter;
-
-import javax.ws.rs.core.Application;
-import java.util.HashSet;
-import java.util.Set;
-
-/**
- * Ozone Application.
- */
-public class ObjectStoreApplication extends Application {
- public ObjectStoreApplication() {
- super();
- }
-
- @Override
- public Set<Class<?>> getClasses() {
- HashSet<Class<?>> set = new HashSet<>();
- set.add(BucketHandler.class);
- set.add(VolumeHandler.class);
- set.add(KeyHandler.class);
- set.add(OzoneExceptionMapper.class);
- set.add(LengthInputStreamMessageBodyWriter.class);
- set.add(StringMessageBodyWriter.class);
- return set;
- }
-
- @Override
- public Set<Object> getSingletons() {
- HashSet<Object> set = new HashSet<>();
- set.add(ServiceFilter.class);
- return set;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
deleted file mode 100644
index b6a8fda..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/OzoneHttpServer.java
+++ /dev/null
@@ -1,216 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.web;
-
-import com.google.common.base.Optional;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.eclipse.jetty.webapp.WebAppContext;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.servlet.http.HttpServlet;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-/**
- * Base class for HTTP server of the Ozone related components.
- */
-public abstract class OzoneHttpServer {
-
- private static final Logger LOG =
- LoggerFactory.getLogger(OzoneHttpServer.class);
-
- private HttpServer2 httpServer;
- private final Configuration conf;
-
- private InetSocketAddress httpAddress;
- private InetSocketAddress httpsAddress;
-
- private HttpConfig.Policy policy;
-
- private String name;
-
- public OzoneHttpServer(Configuration conf, String name) throws IOException {
- this.name = name;
- this.conf = conf;
- if (isEnabled()) {
- policy = DFSUtil.getHttpPolicy(conf);
- if (policy.isHttpEnabled()) {
- this.httpAddress = getHttpBindAddress();
- }
- if (policy.isHttpsEnabled()) {
- this.httpsAddress = getHttpsBindAddress();
- }
- HttpServer2.Builder builder = null;
- builder = DFSUtil.httpServerTemplateForNNAndJN(conf, this.httpAddress,
- this.httpsAddress, name, getSpnegoPrincipal(), getKeytabFile());
-
- final boolean xFrameEnabled = conf.getBoolean(
- DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED,
- DFSConfigKeys.DFS_XFRAME_OPTION_ENABLED_DEFAULT);
-
- final String xFrameOptionValue = conf.getTrimmed(
- DFSConfigKeys.DFS_XFRAME_OPTION_VALUE,
- DFSConfigKeys.DFS_XFRAME_OPTION_VALUE_DEFAULT);
-
- builder.configureXFrame(xFrameEnabled).setXFrameOption(xFrameOptionValue);
-
- httpServer = builder.build();
-
- }
-
- }
-
- /**
- * Add a servlet to OzoneHttpServer.
- * @param servletName The name of the servlet
- * @param pathSpec The path spec for the servlet
- * @param clazz The servlet class
- */
- protected void addServlet(String servletName, String pathSpec,
- Class<? extends HttpServlet> clazz) {
- httpServer.addServlet(servletName, pathSpec, clazz);
- }
-
- /**
- * Returns the WebAppContext associated with this HttpServer.
- * @return WebAppContext
- */
- protected WebAppContext getWebAppContext() {
- return httpServer.getWebAppContext();
- }
-
- protected InetSocketAddress getBindAddress(String bindHostKey,
- String addressKey, String bindHostDefault, int bindPortdefault) {
- final Optional<String> bindHost =
- OzoneClientUtils.getHostNameFromConfigKeys(conf, bindHostKey);
-
- final Optional<Integer> addressPort =
- OzoneClientUtils.getPortNumberFromConfigKeys(conf, addressKey);
-
- final Optional<String> addresHost =
- OzoneClientUtils.getHostNameFromConfigKeys(conf, addressKey);
-
- String hostName = bindHost.or(addresHost).or(bindHostDefault);
-
- return NetUtils.createSocketAddr(
- hostName + ":" + addressPort.or(bindPortdefault));
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the HTTPS web interface.
- *
- * @return Target InetSocketAddress for the Ozone HTTPS endpoint.
- */
- public InetSocketAddress getHttpsBindAddress() {
- return getBindAddress(getHttpsBindHostKey(), getHttpsAddressKey(),
- getBindHostDefault(), getHttpsBindPortDefault());
- }
-
- /**
- * Retrieve the socket address that should be used by clients to connect
- * to the HTTP web interface.
- * <p>
- * * @return Target InetSocketAddress for the Ozone HTTP endpoint.
- */
- public InetSocketAddress getHttpBindAddress() {
- return getBindAddress(getHttpBindHostKey(), getHttpAddressKey(),
- getBindHostDefault(), getHttpBindPortDefault());
-
- }
-
- public void start() throws IOException {
- if (httpServer != null && isEnabled()) {
- httpServer.start();
- updateConnectorAddress();
- }
-
- }
-
- private boolean isEnabled() {
- return conf.getBoolean(getEnabledKey(), true);
- }
-
- public void stop() throws Exception {
- if (httpServer != null) {
- httpServer.stop();
- }
- }
-
- /**
- * Update the configured listen address based on the real port
- * <p>
- * (eg. replace :0 with real port)
- */
- public void updateConnectorAddress() {
- int connIdx = 0;
- if (policy.isHttpEnabled()) {
- httpAddress = httpServer.getConnectorAddress(connIdx++);
- String realAddress = NetUtils.getHostPortString(httpAddress);
- conf.set(getHttpAddressKey(), realAddress);
- LOG.info(
- String.format("HTTP server of %s is listening at http://%s",
- name.toUpperCase(), realAddress));
- }
-
- if (policy.isHttpsEnabled()) {
- httpsAddress = httpServer.getConnectorAddress(connIdx);
- String realAddress = NetUtils.getHostPortString(httpsAddress);
- conf.set(getHttpsAddressKey(), realAddress);
- LOG.info(
- String.format("HTTP server of %s is listening at https://%s",
- name.toUpperCase(), realAddress));
- }
- }
-
- public InetSocketAddress getHttpAddress() {
- return httpAddress;
- }
-
- public InetSocketAddress getHttpsAddress() {
- return httpsAddress;
- }
-
- protected abstract String getHttpAddressKey();
-
- protected abstract String getHttpsAddressKey();
-
- protected abstract String getHttpBindHostKey();
-
- protected abstract String getHttpsBindHostKey();
-
- protected abstract String getBindHostDefault();
-
- protected abstract int getHttpBindPortDefault();
-
- protected abstract int getHttpsBindPortDefault();
-
- protected abstract String getKeytabFile();
-
- protected abstract String getSpnegoPrincipal();
-
- protected abstract String getEnabledKey();
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
deleted file mode 100644
index 9cfd702..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.js
+++ /dev/null
@@ -1,11 +0,0 @@
-/* nvd3 version 1.8.5 (https://github.com/novus/nvd3) 2016-12-01 */
-
-!function(){var a={};a.dev=!1,a.tooltip=a.tooltip||{},a.utils=a.utils||{},a.models=a.models||{},a.charts={},a.logs={},a.dom={},"undefined"!=typeof module&&"undefined"!=typeof exports&&"undefined"==typeof d3&&(d3=require("d3")),a.dispatch=d3.dispatch("render_start","render_end"),Function.prototype.bind||(Function.prototype.bind=function(a){if("function"!=typeof this)throw new TypeError("Function.prototype.bind - what is trying to be bound is not callable");var b=Array.prototype.slice.call(arguments,1),c=this,d=function(){},e=function(){return c.apply(this instanceof d&&a?this:a,b.concat(Array.prototype.slice.call(arguments)))};return d.prototype=this.prototype,e.prototype=new d,e}),a.dev&&(a.dispatch.on("render_start",function(b){a.logs.startTime=+new Date}),a.dispatch.on("render_end",function(b){a.logs.endTime=+new Date,a.logs.totalTime=a.logs.endTime-a.logs.startTime,a.log("total",a.logs.totalTime)})),a.log=function(){if(a.dev&&window.console&&console.log&&console.log.apply)console
.log.apply(console,arguments);else if(a.dev&&window.console&&"function"==typeof console.log&&Function.prototype.bind){var b=Function.prototype.bind.call(console.log,console);b.apply(console,arguments)}return arguments[arguments.length-1]},a.deprecated=function(a,b){console&&console.warn&&console.warn("nvd3 warning: `"+a+"` has been deprecated. ",b||"")},a.render=function(b){b=b||1,a.render.active=!0,a.dispatch.render_start();var c=function(){for(var d,e,f=0;b>f&&(e=a.render.queue[f]);f++)d=e.generate(),typeof e.callback==typeof Function&&e.callback(d);a.render.queue.splice(0,f),a.render.queue.length?setTimeout(c):(a.dispatch.render_end(),a.render.active=!1)};setTimeout(c)},a.render.active=!1,a.render.queue=[],a.addGraph=function(b){typeof arguments[0]==typeof Function&&(b={generate:arguments[0],callback:arguments[1]}),a.render.queue.push(b),a.render.active||a.render()},"undefined"!=typeof module&&"undefined"!=typeof exports&&(module.exports=a),"undefined"!=typeof window&&(window.nv=
a),a.dom.write=function(a){return void 0!==window.fastdom?fastdom.mutate(a):a()},a.dom.read=function(a){return void 0!==window.fastdom?fastdom.measure(a):a()},a.interactiveGuideline=function(){"use strict";function b(l){l.each(function(l){function m(){var a=d3.mouse(this),d=a[0],e=a[1],h=!0,i=!1;if(k&&(d=d3.event.offsetX,e=d3.event.offsetY,"svg"!==d3.event.target.tagName&&(h=!1),d3.event.target.className.baseVal.match("nv-legend")&&(i=!0)),h&&(d-=c.left,e-=c.top),"mouseout"===d3.event.type||0>d||0>e||d>o||e>p||d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement||i){if(k&&d3.event.relatedTarget&&void 0===d3.event.relatedTarget.ownerSVGElement&&(void 0===d3.event.relatedTarget.className||d3.event.relatedTarget.className.match(j.nvPointerEventsClass)))return;return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0)}j.hidden(!1);var l="function"==typeof f.rangeBands,m=void 0;if(l){var n=d3.bisect(f.range(),d)-1;if(!(f.range()[n]+f.rangeB
and()>=d))return g.elementMouseout({mouseX:d,mouseY:e}),b.renderGuideLine(null),void j.hidden(!0);m=f.domain()[d3.bisect(f.range(),d)-1]}else m=f.invert(d);g.elementMousemove({mouseX:d,mouseY:e,pointXValue:m}),"dblclick"===d3.event.type&&g.elementDblclick({mouseX:d,mouseY:e,pointXValue:m}),"click"===d3.event.type&&g.elementClick({mouseX:d,mouseY:e,pointXValue:m}),"mousedown"===d3.event.type&&g.elementMouseDown({mouseX:d,mouseY:e,pointXValue:m}),"mouseup"===d3.event.type&&g.elementMouseUp({mouseX:d,mouseY:e,pointXValue:m})}var n=d3.select(this),o=d||960,p=e||400,q=n.selectAll("g.nv-wrap.nv-interactiveLineLayer").data([l]),r=q.enter().append("g").attr("class"," nv-wrap nv-interactiveLineLayer");r.append("g").attr("class","nv-interactiveGuideLine"),i&&(i.on("touchmove",m).on("mousemove",m,!0).on("mouseout",m,!0).on("mousedown",m,!0).on("mouseup",m,!0).on("dblclick",m).on("click",m),b.guideLine=null,b.renderGuideLine=function(c){h&&(b.guideLine&&b.guideLine.attr("x1")===c||a.dom.write(f
unction(){var b=q.select(".nv-interactiveGuideLine").selectAll("line").data(null!=c?[a.utils.NaNtoZero(c)]:[],String);b.enter().append("line").attr("class","nv-guideline").attr("x1",function(a){return a}).attr("x2",function(a){return a}).attr("y1",p).attr("y2",0),b.exit().remove()}))})})}var c={left:0,top:0},d=null,e=null,f=d3.scale.linear(),g=d3.dispatch("elementMousemove","elementMouseout","elementClick","elementDblclick","elementMouseDown","elementMouseUp"),h=!0,i=null,j=a.models.tooltip(),k=window.ActiveXObject;return j.duration(0).hideDelay(0).hidden(!1),b.dispatch=g,b.tooltip=j,b.margin=function(a){return arguments.length?(c.top="undefined"!=typeof a.top?a.top:c.top,c.left="undefined"!=typeof a.left?a.left:c.left,b):c},b.width=function(a){return arguments.length?(d=a,b):d},b.height=function(a){return arguments.length?(e=a,b):e},b.xScale=function(a){return arguments.length?(f=a,b):f},b.showGuideLine=function(a){return arguments.length?(h=a,b):h},b.svgContainer=function(a){retur
n arguments.length?(i=a,b):i},b},a.interactiveBisect=function(a,b,c){"use strict";if(!(a instanceof Array))return null;var d;d="function"!=typeof c?function(a){return a.x}:c;var e=function(a,b){return d(a)-b},f=d3.bisector(e).left,g=d3.max([0,f(a,b)-1]),h=d(a[g]);if("undefined"==typeof h&&(h=g),h===b)return g;var i=d3.min([g+1,a.length-1]),j=d(a[i]);return"undefined"==typeof j&&(j=i),Math.abs(j-b)>=Math.abs(h-b)?g:i},a.nearestValueIndex=function(a,b,c){"use strict";var d=1/0,e=null;return a.forEach(function(a,f){var g=Math.abs(b-a);null!=a&&d>=g&&c>g&&(d=g,e=f)}),e},a.models.tooltip=function(){"use strict";function b(){if(!l||!l.node()){var a=[1];l=d3.select(document.body).select("#"+d).data(a),l.enter().append("div").attr("class","nvtooltip "+(i?i:"xy-tooltip")).attr("id",d).style("top",0).style("left",0).style("opacity",0).style("position","fixed").selectAll("div, table, td, tr").classed(q,!0).classed(q,!0),l.exit().remove()}}function c(){return n&&w(e)?(a.dom.write(function(){b()
;var a=u(e);a&&(l.node().innerHTML=a),y()}),c):void 0}var d="nvtooltip-"+Math.floor(1e5*Math.random()),e=null,f="w",g=25,h=0,i=null,j=!0,k=200,l=null,m={left:null,top:null},n=!0,o=100,p=!0,q="nv-pointer-events-none",r=function(a,b){return a},s=function(a){return a},t=function(a,b){return a},u=function(a){if(null===a)return"";var b=d3.select(document.createElement("table"));if(p){var c=b.selectAll("thead").data([a]).enter().append("thead");c.append("tr").append("td").attr("colspan",3).append("strong").classed("x-value",!0).html(s(a.value))}var d=b.selectAll("tbody").data([a]).enter().append("tbody"),e=d.selectAll("tr").data(function(a){return a.series}).enter().append("tr").classed("highlight",function(a){return a.highlight});e.append("td").classed("legend-color-guide",!0).append("div").style("background-color",function(a){return a.color}),e.append("td").classed("key",!0).classed("total",function(a){return!!a.total}).html(function(a,b){return t(a.key,b)}),e.append("td").classed("valu
e",!0).html(function(a,b){return r(a.value,b)}),e.filter(function(a,b){return void 0!==a.percent}).append("td").classed("percent",!0).html(function(a,b){return"("+d3.format("%")(a.percent)+")"}),e.selectAll("td").each(function(a){if(a.highlight){var b=d3.scale.linear().domain([0,1]).range(["#fff",a.color]),c=.6;d3.select(this).style("border-bottom-color",b(c)).style("border-top-color",b(c))}});var f=b.node().outerHTML;return void 0!==a.footer&&(f+="<div class='footer'>"+a.footer+"</div>"),f},v=function(){var a={left:null!==d3.event?d3.event.clientX:0,top:null!==d3.event?d3.event.clientY:0};if("none"!=getComputedStyle(document.body).transform){var b=document.body.getBoundingClientRect();a.left-=b.left,a.top-=b.top}return a},w=function(b){if(b&&b.series){if(a.utils.isArray(b.series))return!0;if(a.utils.isObject(b.series))return b.series=[b.series],!0}return!1},x=function(a){var b,c,d,e=l.node().offsetHeight,h=l.node().offsetWidth,i=document.documentElement.clientWidth,j=document.docum
entElement.clientHeight;switch(f){case"e":b=-h-g,c=-(e/2),a.left+b<0&&(b=g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"w":b=g,c=-(e/2),a.left+b+h>i&&(b=-h-g),(d=a.top+c)<0&&(c-=d),(d=a.top+c+e)>j&&(c-=d-j);break;case"n":b=-(h/2)-5,c=g,a.top+c+e>j&&(c=-e-g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"s":b=-(h/2),c=-e-g,a.top+c<0&&(c=g),(d=a.left+b)<0&&(b-=d),(d=a.left+b+h)>i&&(b-=d-i);break;case"center":b=-(h/2),c=-(e/2);break;default:b=0,c=0}return{left:b,top:c}},y=function(){a.dom.read(function(){var a=v(),b=x(a),c=a.left+b.left,d=a.top+b.top;if(j)l.interrupt().transition().delay(k).duration(0).style("opacity",0);else{var e="translate("+m.left+"px, "+m.top+"px)",f="translate("+Math.round(c)+"px, "+Math.round(d)+"px)",g=d3.interpolateString(e,f),h=l.style("opacity")<.1;l.interrupt().transition().duration(h?0:o).styleTween("transform",function(a){return g},"important").styleTween("-webkit-transform",function(a){return g}).style("-ms-transform",f).styl
e("opacity",1)}m.left=c,m.top=d})};return c.nvPointerEventsClass=q,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{duration:{get:function(){return o},set:function(a){o=a}},gravity:{get:function(){return f},set:function(a){f=a}},distance:{get:function(){return g},set:function(a){g=a}},snapDistance:{get:function(){return h},set:function(a){h=a}},classes:{get:function(){return i},set:function(a){i=a}},enabled:{get:function(){return n},set:function(a){n=a}},hideDelay:{get:function(){return k},set:function(a){k=a}},contentGenerator:{get:function(){return u},set:function(a){u=a}},valueFormatter:{get:function(){return r},set:function(a){r=a}},headerFormatter:{get:function(){return s},set:function(a){s=a}},keyFormatter:{get:function(){return t},set:function(a){t=a}},headerEnabled:{get:function(){return p},set:function(a){p=a}},position:{get:function(){return v},set:function(a){v=a}},chartContainer:{get:function(){return document.body},set:function(b){a.deprecated("chartCo
ntainer","feature removed after 1.8.3")}},fixedTop:{get:function(){return null},set:function(b){a.deprecated("fixedTop","feature removed after 1.8.1")}},offset:{get:function(){return{left:0,top:0}},set:function(b){a.deprecated("offset","use chart.tooltip.distance() instead")}},hidden:{get:function(){return j},set:function(a){j!=a&&(j=!!a,c())}},data:{get:function(){return e},set:function(a){a.point&&(a.value=a.point.x,a.series=a.series||{},a.series.value=a.point.y,a.series.color=a.point.color||a.series.color),e=a}},node:{get:function(){return l.node()},set:function(a){}},id:{get:function(){return d},set:function(a){}}}),a.utils.initOptions(c),c},a.utils.windowSize=function(){var a={width:640,height:480};return window.innerWidth&&window.innerHeight?(a.width=window.innerWidth,a.height=window.innerHeight,a):"CSS1Compat"==document.compatMode&&document.documentElement&&document.documentElement.offsetWidth?(a.width=document.documentElement.offsetWidth,a.height=document.documentElement.off
setHeight,a):document.body&&document.body.offsetWidth?(a.width=document.body.offsetWidth,a.height=document.body.offsetHeight,a):a},a.utils.isArray=Array.isArray,a.utils.isObject=function(a){return null!==a&&"object"==typeof a},a.utils.isFunction=function(a){return"function"==typeof a},a.utils.isDate=function(a){return"[object Date]"===toString.call(a)},a.utils.isNumber=function(a){return!isNaN(a)&&"number"==typeof a},a.utils.windowResize=function(b){return window.addEventListener?window.addEventListener("resize",b):a.log("ERROR: Failed to bind to window.resize with: ",b),{callback:b,clear:function(){window.removeEventListener("resize",b)}}},a.utils.getColor=function(b){if(void 0===b)return a.utils.defaultColor();if(a.utils.isArray(b)){var c=d3.scale.ordinal().range(b);return function(a,b){var d=void 0===b?a:b;return a.color||c(d)}}return b},a.utils.defaultColor=function(){return a.utils.getColor(d3.scale.category20().range())},a.utils.customTheme=function(b,c,d){c=c||function(a){ret
urn a.key},d=d||d3.scale.category20().range();var e=d.length;return function(f,g){var h=c(f);return a.utils.isFunction(b[h])?b[h]():void 0!==b[h]?b[h]:(e||(e=d.length),e-=1,d[e])}},a.utils.pjax=function(b,c){var d=function(d){d3.html(d,function(d){var e=d3.select(c).node();e.parentNode.replaceChild(d3.select(d).select(c).node(),e),a.utils.pjax(b,c)})};d3.selectAll(b).on("click",function(){history.pushState(this.href,this.textContent,this.href),d(this.href),d3.event.preventDefault()}),d3.select(window).on("popstate",function(){d3.event.state&&d(d3.event.state)})},a.utils.calcApproxTextWidth=function(b){if(a.utils.isFunction(b.style)&&a.utils.isFunction(b.text)){var c=parseInt(b.style("font-size").replace("px",""),10),d=b.text().length;return a.utils.NaNtoZero(d*c*.5)}return 0},a.utils.NaNtoZero=function(b){return!a.utils.isNumber(b)||isNaN(b)||null===b||b===1/0||b===-(1/0)?0:b},d3.selection.prototype.watchTransition=function(a){var b=[this].concat([].slice.call(arguments,1));return a
.transition.apply(a,b)},a.utils.renderWatch=function(b,c){if(!(this instanceof a.utils.renderWatch))return new a.utils.renderWatch(b,c);var d=void 0!==c?c:250,e=[],f=this;this.models=function(a){return a=[].slice.call(arguments,0),a.forEach(function(a){a.__rendered=!1,function(a){a.dispatch.on("renderEnd",function(b){a.__rendered=!0,f.renderEnd("model")})}(a),e.indexOf(a)<0&&e.push(a)}),this},this.reset=function(a){void 0!==a&&(d=a),e=[]},this.transition=function(a,b,c){if(b=arguments.length>1?[].slice.call(arguments,1):[],c=b.length>1?b.pop():void 0!==d?d:250,a.__rendered=!1,e.indexOf(a)<0&&e.push(a),0===c)return a.__rendered=!0,a.delay=function(){return this},a.duration=function(){return this},a;0===a.length?a.__rendered=!0:a.every(function(a){return!a.length})?a.__rendered=!0:a.__rendered=!1;var g=0;return a.transition().duration(c).each(function(){++g}).each("end",function(c,d){0===--g&&(a.__rendered=!0,f.renderEnd.apply(this,b))})},this.renderEnd=function(){e.every(function(a){
return a.__rendered})&&(e.forEach(function(a){a.__rendered=!1}),b.renderEnd.apply(this,arguments))}},a.utils.deepExtend=function(b){var c=arguments.length>1?[].slice.call(arguments,1):[];c.forEach(function(c){for(var d in c){var e=a.utils.isArray(b[d]),f=a.utils.isObject(b[d]),g=a.utils.isObject(c[d]);f&&!e&&g?a.utils.deepExtend(b[d],c[d]):b[d]=c[d]}})},a.utils.state=function(){if(!(this instanceof a.utils.state))return new a.utils.state;var b={},c=function(){},d=function(){return{}},e=null,f=null;this.dispatch=d3.dispatch("change","set"),this.dispatch.on("set",function(a){c(a,!0)}),this.getter=function(a){return d=a,this},this.setter=function(a,b){return b||(b=function(){}),c=function(c,d){a(c),d&&b()},this},this.init=function(b){e=e||{},a.utils.deepExtend(e,b)};var g=function(){var a=d();if(JSON.stringify(a)===JSON.stringify(b))return!1;for(var c in a)void 0===b[c]&&(b[c]={}),b[c]=a[c],f=!0;return!0};this.update=function(){e&&(c(e,!1),e=null),g.call(this)&&this.dispatch.change(b)}
},a.utils.optionsFunc=function(b){return b&&d3.map(b).forEach(function(b,c){a.utils.isFunction(this[b])&&this[b](c)}.bind(this)),this},a.utils.calcTicksX=function(b,c){var d=1,e=0;for(e;e<c.length;e+=1){var f=c[e]&&c[e].values?c[e].values.length:0;d=f>d?f:d}return a.log("Requested number of ticks: ",b),a.log("Calculated max values to be: ",d),b=b>d?b=d-1:b,b=1>b?1:b,b=Math.floor(b),a.log("Calculating tick count as: ",b),b},a.utils.calcTicksY=function(b,c){return a.utils.calcTicksX(b,c)},a.utils.initOption=function(a,b){a._calls&&a._calls[b]?a[b]=a._calls[b]:(a[b]=function(c){return arguments.length?(a._overrides[b]=!0,a._options[b]=c,a):a._options[b]},a["_"+b]=function(c){return arguments.length?(a._overrides[b]||(a._options[b]=c),a):a._options[b]})},a.utils.initOptions=function(b){b._overrides=b._overrides||{};var c=Object.getOwnPropertyNames(b._options||{}),d=Object.getOwnPropertyNames(b._calls||{});c=c.concat(d);for(var e in c)a.utils.initOption(b,c[e])},a.utils.inheritOptionsD3=
function(a,b,c){a._d3options=c.concat(a._d3options||[]),c.unshift(b),c.unshift(a),d3.rebind.apply(this,c)},a.utils.arrayUnique=function(a){return a.sort().filter(function(b,c){return!c||b!=a[c-1]})},a.utils.symbolMap=d3.map(),a.utils.symbol=function(){function b(b,e){var f=c.call(this,b,e),g=d.call(this,b,e);return-1!==d3.svg.symbolTypes.indexOf(f)?d3.svg.symbol().type(f).size(g)():a.utils.symbolMap.get(f)(g)}var c,d=64;return b.type=function(a){return arguments.length?(c=d3.functor(a),b):c},b.size=function(a){return arguments.length?(d=d3.functor(a),b):d},b},a.utils.inheritOptions=function(b,c){var d=Object.getOwnPropertyNames(c._options||{}),e=Object.getOwnPropertyNames(c._calls||{}),f=c._inherited||[],g=c._d3options||[],h=d.concat(e).concat(f).concat(g);h.unshift(c),h.unshift(b),d3.rebind.apply(this,h),b._inherited=a.utils.arrayUnique(d.concat(e).concat(f).concat(d).concat(b._inherited||[])),b._d3options=a.utils.arrayUnique(g.concat(b._d3options||[]))},a.utils.initSVG=function(a)
{a.classed({"nvd3-svg":!0})},a.utils.sanitizeHeight=function(a,b){return a||parseInt(b.style("height"),10)||400},a.utils.sanitizeWidth=function(a,b){return a||parseInt(b.style("width"),10)||960},a.utils.availableHeight=function(b,c,d){return Math.max(0,a.utils.sanitizeHeight(b,c)-d.top-d.bottom)},a.utils.availableWidth=function(b,c,d){return Math.max(0,a.utils.sanitizeWidth(b,c)-d.left-d.right)},a.utils.noData=function(b,c){var d=b.options(),e=d.margin(),f=d.noData(),g=null==f?["No Data Available."]:[f],h=a.utils.availableHeight(null,c,e),i=a.utils.availableWidth(null,c,e),j=e.left+i/2,k=e.top+h/2;c.selectAll("g").remove();var l=c.selectAll(".nv-noData").data(g);l.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),l.attr("x",j).attr("y",k).text(function(a){return a})},a.utils.wrapTicks=function(a,b){a.each(function(){for(var a,c=d3.select(this),d=c.text().split(/\s+/).reverse(),e=[],f=0,g=1.1,h=c.attr("y"),i=parseFloat(c.attr("dy")
),j=c.text(null).append("tspan").attr("x",0).attr("y",h).attr("dy",i+"em");a=d.pop();)e.push(a),j.text(e.join(" ")),j.node().getComputedTextLength()>b&&(e.pop(),j.text(e.join(" ")),e=[a],j=c.append("tspan").attr("x",0).attr("y",h).attr("dy",++f*g+i+"em").text(a))})},a.utils.arrayEquals=function(b,c){if(b===c)return!0;if(!b||!c)return!1;if(b.length!=c.length)return!1;for(var d=0,e=b.length;e>d;d++)if(b[d]instanceof Array&&c[d]instanceof Array){if(!a.arrayEquals(b[d],c[d]))return!1}else if(b[d]!=c[d])return!1;return!0},a.models.axis=function(){"use strict";function b(g){return t.reset(),g.each(function(b){var g=d3.select(this);a.utils.initSVG(g);var q=g.selectAll("g.nv-wrap.nv-axis").data([b]),r=q.enter().append("g").attr("class","nvd3 nv-wrap nv-axis"),u=(r.append("g"),q.select("g"));null!==n?c.ticks(n):("top"==c.orient()||"bottom"==c.orient())&&c.ticks(Math.abs(d.range()[1]-d.range()[0])/100),u.watchTransition(t,"axis").call(c),s=s||c.scale();var v=c.tickFormat();null==v&&(v=s.tickF
ormat());var w=u.selectAll("text.nv-axislabel").data([h||null]);w.exit().remove(),void 0!==p&&u.selectAll("g").select("text").style("font-size",p);var x,y,z;switch(c.orient()){case"top":w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",0).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b))+",0)"}).select("text").attr("dy","-0.5em").attr("y",-c.tickPadding()).attr("text-anchor","middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition
(t,"min-max top").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d.range()[c])+",0)"}));break;case"bottom":x=o+36;var A=30,B=0,C=u.selectAll("g").select("text"),D="";if(j%360){C.attr("transform",""),C.each(function(a,b){var c=this.getBoundingClientRect(),d=c.width;B=c.height,d>A&&(A=d)}),D="rotate("+j+" 0,"+(B/2+c.tickPadding())+")";var E=Math.abs(Math.sin(j*Math.PI/180));x=(E?E*A:A)+30,C.attr("transform",D).style("text-anchor",j%360>0?"start":"end")}else l?C.attr("transform",function(a,b){return"translate(0,"+(b%2==0?"0":"12")+")"}):C.attr("transform","translate(0,0)");w.enter().append("text").attr("class","nv-axislabel"),z=0,1===d.range().length?z=m?2*d.range()[0]+d.rangeBand():0:2===d.range().length?z=m?d.range()[0]+d.range()[1]+d.rangeBand():d.range()[1]:d.range().length>2&&(z=d.range()[d.range().length-1]+(d.range()[1]-d.range()[0])),w.attr("text-anchor","middle").attr("y",x).attr("x",z/2),i&&(y=q.selectAll("g.nv-axisMaxMin").data([d.domain()[0],d.domain()[
d.domain().length-1]]),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-x",0==b?"nv-axisMin-x":"nv-axisMax-x"].join(" ")}).append("text"),y.exit().remove(),y.attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}).select("text").attr("dy",".71em").attr("y",c.tickPadding()).attr("transform",D).style("text-anchor",j?j%360>0?"start":"end":"middle").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max bottom").attr("transform",function(b,c){return"translate("+a.utils.NaNtoZero(d(b)+(m?d.rangeBand()/2:0))+",0)"}));break;case"right":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"begin").attr("transform",k?"rotate(90)":"").attr("y",k?-Math.max(e.right,f)+12-(o||0):-10).attr("x",k?d3.max(d.range())/2:c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["n
v-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").style("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",c.tickPadding()).style("text-anchor","start").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1));break;case"left":w.enter().append("text").attr("class","nv-axislabel"),w.style("text-anchor",k?"middle":"end").attr("transform",k?"rotate(-90)":"").attr("y",k?-Math.max(e.left,f)+25-(o||0):-10).attr("x",k?-d3.max(d.range())/2:-c.tickPadding()),i&&(y=q.selectAll("g.nv-axisMaxMin").data(d.domain()),y.enter().append("g").attr("class",function(a,b){return["nv-axisMaxMin","nv-axisMaxMin-y",0==b?"nv-axisMin-y":"nv-axisMax-y"].join(" ")}).append("text").styl
e("opacity",0),y.exit().remove(),y.attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(s(b))+")"}).select("text").attr("dy",".32em").attr("y",0).attr("x",-c.tickPadding()).attr("text-anchor","end").text(function(a,b){var c=v(a);return(""+c).match("NaN")?"":c}),y.watchTransition(t,"min-max right").attr("transform",function(b,c){return"translate(0,"+a.utils.NaNtoZero(d.range()[c])+")"}).select("text").style("opacity",1))}if(w.text(function(a){return a}),!i||"left"!==c.orient()&&"right"!==c.orient()||(u.selectAll("g").each(function(a,b){d3.select(this).select("text").attr("opacity",1),(d(a)<d.range()[1]+10||d(a)>d.range()[0]-10)&&((a>1e-10||-1e-10>a)&&d3.select(this).attr("opacity",0),d3.select(this).select("text").attr("opacity",0))}),d.domain()[0]==d.domain()[1]&&0==d.domain()[0]&&q.selectAll("g.nv-axisMaxMin").style("opacity",function(a,b){return b?0:1})),i&&("top"===c.orient()||"bottom"===c.orient())){var F=[];q.selectAll("g.nv-axisMaxMin").each(function(a,b){try{
b?F.push(d(a)-this.getBoundingClientRect().width-4):F.push(d(a)+this.getBoundingClientRect().width+4)}catch(c){b?F.push(d(a)-4):F.push(d(a)+4)}}),u.selectAll("g").each(function(a,b){(d(a)<F[0]||d(a)>F[1])&&(a>1e-10||-1e-10>a?d3.select(this).remove():d3.select(this).select("text").remove())})}u.selectAll(".tick").filter(function(a){return!parseFloat(Math.round(1e5*a)/1e6)&&void 0!==a}).classed("zero",!0),s=d.copy()}),t.renderEnd("axis immediate"),b}var c=d3.svg.axis(),d=d3.scale.linear(),e={top:0,right:0,bottom:0,left:0},f=75,g=60,h=null,i=!0,j=0,k=!0,l=!1,m=!1,n=null,o=0,p=void 0,q=250,r=d3.dispatch("renderEnd");c.scale(d).orient("bottom").tickFormat(function(a){return a});var s,t=a.utils.renderWatch(r,q);return b.axis=c,b.dispatch=r,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{axisLabelDistance:{get:function(){return o},set:function(a){o=a}},staggerLabels:{get:function(){return l},set:function(a){l=a}},rotateLabels:{get:function(){return j},set:function(a){j=a
}},rotateYLabel:{get:function(){return k},set:function(a){k=a}},showMaxMin:{get:function(){return i},set:function(a){i=a}},axisLabel:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return g},set:function(a){g=a}},ticks:{get:function(){return n},set:function(a){n=a}},width:{get:function(){return f},set:function(a){f=a}},fontSize:{get:function(){return p},set:function(a){p=a}},margin:{get:function(){return e},set:function(a){e.top=void 0!==a.top?a.top:e.top,e.right=void 0!==a.right?a.right:e.right,e.bottom=void 0!==a.bottom?a.bottom:e.bottom,e.left=void 0!==a.left?a.left:e.left}},duration:{get:function(){return q},set:function(a){q=a,t.reset(q)}},scale:{get:function(){return d},set:function(e){d=e,c.scale(d),m="function"==typeof d.rangeBands,a.utils.inheritOptionsD3(b,d,["domain","range","rangeBand","rangeBands"])}}}),a.utils.initOptions(b),a.utils.inheritOptionsD3(b,c,["orient","tickValues","tickSubdivide","tickSize","tickPadding","tickFormat"]),a.utils.inherit
OptionsD3(b,d,["domain","range","rangeBand","rangeBands"]),b},a.models.boxPlot=function(){"use strict";function b(l){return E.reset(),l.each(function(b){var l=j-i.left-i.right,F=k-i.top-i.bottom;A=d3.select(this),a.utils.initSVG(A),m.domain(c||b.map(function(a,b){return o(a,b)})).rangeBands(d||[0,l],.1);var G=[];if(!e){var H,I,J=[];b.forEach(function(a,b){var c=p(a),d=r(a),e=s(a),f=t(a),g=v(a);g&&g.forEach(function(a,b){J.push(w(a,b,void 0))}),e&&J.push(e),c&&J.push(c),d&&J.push(d),f&&J.push(f)}),H=d3.min(J),I=d3.max(J),G=[H,I]}n.domain(e||G),n.range(f||[F,0]),g=g||m,h=h||n.copy().range([n(0),n(0)]);var K=A.selectAll("g.nv-wrap").data([b]);K.enter().append("g").attr("class","nvd3 nv-wrap");K.attr("transform","translate("+i.left+","+i.top+")");var L=K.selectAll(".nv-boxplot").data(function(a){return a}),M=L.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6);L.attr("class","nv-boxplot").attr("transform",function(a,b,c){return"translate("+(m(o(a,b))+.05*m.range
Band())+", 0)"}).classed("hover",function(a){return a.hover}),L.watchTransition(E,"nv-boxplot: boxplots").style("stroke-opacity",1).style("fill-opacity",.75).delay(function(a,c){return c*C/b.length}).attr("transform",function(a,b){return"translate("+(m(o(a,b))+.05*m.rangeBand())+", 0)"}),L.exit().remove(),M.each(function(a,b){var c=d3.select(this);[s,t].forEach(function(d){if(void 0!==d(a)&&null!==d(a)){var e=d===s?"low":"high";c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-whisker nv-boxplot-"+e),c.append("line").style("stroke",u(a)||z(a,b)).attr("class","nv-boxplot-tick nv-boxplot-"+e)}})});var N=function(){return null===D?.9*m.rangeBand():Math.min(75,.9*m.rangeBand())},O=function(){return.45*m.rangeBand()-N()/2},P=function(){return.45*m.rangeBand()+N()/2};[s,t].forEach(function(a){var b=a===s?"low":"high",c=a===s?p:r;L.select("line.nv-boxplot-whisker.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",.45*m.rangeBand()).attr("y1",function(b
,c){return n(a(b))}).attr("x2",.45*m.rangeBand()).attr("y2",function(a,b){return n(c(a))}),L.select("line.nv-boxplot-tick.nv-boxplot-"+b).watchTransition(E,"nv-boxplot: boxplots").attr("x1",O).attr("y1",function(b,c){return n(a(b))}).attr("x2",P).attr("y2",function(b,c){return n(a(b))})}),[s,t].forEach(function(a){var b=a===s?"low":"high";M.selectAll(".nv-boxplot-"+b).on("mouseover",function(b,c,d){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mouseout",function(b,c,d){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:a(b),color:u(b)||z(b,d)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})})}),M.append("rect").attr("class","nv-boxplot-box").on("mouseover",function(a,b){d3.select(this).classed("hover",!0),B.elementMouseover({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}]
,data:a,index:b,e:d3.event})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),B.elementMouseout({key:o(a),value:o(a),series:[{key:"Q3",value:r(a),color:u(a)||z(a,b)},{key:"Q2",value:q(a),color:u(a)||z(a,b)},{key:"Q1",value:p(a),color:u(a)||z(a,b)}],data:a,index:b,e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),L.select("rect.nv-boxplot-box").watchTransition(E,"nv-boxplot: boxes").attr("y",function(a,b){return n(r(a))}).attr("width",N).attr("x",O).attr("height",function(a,b){return Math.abs(n(r(a))-n(p(a)))||1}).style("fill",function(a,b){return u(a)||z(a,b)}).style("stroke",function(a,b){return u(a)||z(a,b)}),M.append("line").attr("class","nv-boxplot-median"),L.select("line.nv-boxplot-median").watchTransition(E,"nv-boxplot: boxplots line").attr("x1",O).attr("y1",function(a,b){return n(q(a))}).attr("x2",P).attr("y2",function(a,b){return n(q(a))});var Q=L.selectAll(".nv-boxplot-outlier").data(function(a){return v(a)||[]});Q.enter().appen
d("circle").style("fill",function(a,b,c){return y(a,b,c)||z(a,c)}).style("stroke",function(a,b,c){return y(a,b,c)||z(a,c)}).style("z-index",9e3).on("mouseover",function(a,b,c){d3.select(this).classed("hover",!0),B.elementMouseover({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mouseout",function(a,b,c){d3.select(this).classed("hover",!1),B.elementMouseout({series:{key:x(a,b,c),color:y(a,b,c)||z(a,c)},e:d3.event})}).on("mousemove",function(a,b){B.elementMousemove({e:d3.event})}),Q.attr("class","nv-boxplot-outlier"),Q.watchTransition(E,"nv-boxplot: nv-boxplot-outlier").attr("cx",.45*m.rangeBand()).attr("cy",function(a,b,c){return n(w(a,b,c))}).attr("r","3"),Q.exit().remove(),g=m.copy(),h=n.copy()}),E.renderEnd("nv-boxplot immediate"),b}var c,d,e,f,g,h,i={top:0,right:0,bottom:0,left:0},j=960,k=500,l=Math.floor(1e4*Math.random()),m=d3.scale.ordinal(),n=d3.scale.linear(),o=function(a){return a.label},p=function(a){return a.values.Q1},q=function(a){return a.values.Q2},r=
function(a){return a.values.Q3},s=function(a){return a.values.whisker_low},t=function(a){return a.values.whisker_high},u=function(a){return a.color},v=function(a){return a.values.outliers},w=function(a,b,c){return a},x=function(a,b,c){return a},y=function(a,b,c){return void 0},z=a.utils.defaultColor(),A=null,B=d3.dispatch("elementMouseover","elementMouseout","elementMousemove","renderEnd"),C=250,D=null,E=a.utils.renderWatch(B,C);return b.dispatch=B,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return j},set:function(a){j=a}},height:{get:function(){return k},set:function(a){k=a}},maxBoxWidth:{get:function(){return D},set:function(a){D=a}},x:{get:function(){return o},set:function(a){o=a}},q1:{get:function(){return p},set:function(a){p=a}},q2:{get:function(){return q},set:function(a){q=a}},q3:{get:function(){return r},set:function(a){r=a}},wl:{get:function(){return s},set:function(a){s=a}},wh:{get:function(){return t},set:function(a){t=a}},ite
mColor:{get:function(){return u},set:function(a){u=a}},outliers:{get:function(){return v},set:function(a){
-v=a}},outlierValue:{get:function(){return w},set:function(a){w=a}},outlierLabel:{get:function(){return x},set:function(a){x=a}},outlierColor:{get:function(){return y},set:function(a){y=a}},xScale:{get:function(){return m},set:function(a){m=a}},yScale:{get:function(){return n},set:function(a){n=a}},xDomain:{get:function(){return c},set:function(a){c=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return d},set:function(a){d=a}},yRange:{get:function(){return f},set:function(a){f=a}},id:{get:function(){return l},set:function(a){l=a}},y:{get:function(){return console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead."),{}},set:function(a){console.warn("BoxPlot 'y' chart option is deprecated. Please use model overrides instead.")}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.l
eft:i.left}},color:{get:function(){return z},set:function(b){z=a.utils.getColor(b)}},duration:{get:function(){return C},set:function(a){C=a,E.reset(C)}}}),a.utils.initOptions(b),b},a.models.boxPlotChart=function(){"use strict";function b(k){return t.reset(),t.models(e),l&&t.models(f),m&&t.models(g),k.each(function(k){var p=d3.select(this);a.utils.initSVG(p);var t=(i||parseInt(p.style("width"))||960)-h.left-h.right,u=(j||parseInt(p.style("height"))||400)-h.top-h.bottom;if(b.update=function(){r.beforeUpdate(),p.transition().duration(s).call(b)},b.container=this,!k||!k.length){var v=p.selectAll(".nv-noData").data([q]);return v.enter().append("text").attr("class","nvd3 nv-noData").attr("dy","-.7em").style("text-anchor","middle"),v.attr("x",h.left+t/2).attr("y",h.top+u/2).text(function(a){return a}),b}p.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var w=p.selectAll("g.nv-wrap.nv-boxPlotWithAxes").data([k]),x=w.enter().append("g").attr("class","nvd3 nv-wrap nv-boxP
lotWithAxes").append("g"),y=x.append("defs"),z=w.select("g");x.append("g").attr("class","nv-x nv-axis"),x.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),x.append("g").attr("class","nv-barsWrap"),z.attr("transform","translate("+h.left+","+h.top+")"),n&&z.select(".nv-y.nv-axis").attr("transform","translate("+t+",0)"),e.width(t).height(u);var A=z.select(".nv-barsWrap").datum(k.filter(function(a){return!a.disabled}));if(A.transition().call(e),y.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),z.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(o?2:1)).attr("height",16).attr("x",-c.rangeBand()/(o?1:2)),l){f.scale(c).ticks(a.utils.calcTicksX(t/100,k)).tickSize(-u,0),z.select(".nv-x.nv-axis").attr("transform","translate(0,"+d.range()[0]+")"),z.select(".nv-x.nv-axis").call(f);var B=z.select(".nv-x.nv-axis").selectAll("g");o&&B.selectAll("text").attr("transform",function(a,b,c){return"translate(0,"+(c
%2===0?"5":"17")+")"})}m&&(g.scale(d).ticks(Math.floor(u/36)).tickSize(-t,0),z.select(".nv-y.nv-axis").call(g)),z.select(".nv-zeroLine line").attr("x1",0).attr("x2",t).attr("y1",d(0)).attr("y2",d(0))}),t.renderEnd("nv-boxplot chart immediate"),b}var c,d,e=a.models.boxPlot(),f=a.models.axis(),g=a.models.axis(),h={top:15,right:10,bottom:50,left:60},i=null,j=null,k=a.utils.getColor(),l=!0,m=!0,n=!1,o=!1,p=a.models.tooltip(),q="No Data Available.",r=d3.dispatch("beforeUpdate","renderEnd"),s=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(n?"right":"left").tickFormat(d3.format(",.1f")),p.duration(0);var t=a.utils.renderWatch(r,s);return e.dispatch.on("elementMouseover.tooltip",function(a){p.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){p.data(a).hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){p()}),b.dispatch=r,b.boxplot=e,b.xAxis=f,b.yAxis=g,b.tooltip=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.c
reate({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},staggerLabels:{get:function(){return o},set:function(a){o=a}},showXAxis:{get:function(){return l},set:function(a){l=a}},showYAxis:{get:function(){return m},set:function(a){m=a}},tooltipContent:{get:function(){return p},set:function(a){p=a}},noData:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!==a.top?a.top:h.top,h.right=void 0!==a.right?a.right:h.right,h.bottom=void 0!==a.bottom?a.bottom:h.bottom,h.left=void 0!==a.left?a.left:h.left}},duration:{get:function(){return s},set:function(a){s=a,t.reset(s),e.duration(s),f.duration(s),g.duration(s)}},color:{get:function(){return k},set:function(b){k=a.utils.getColor(b),e.color(k)}},rightAlignYAxis:{get:function(){return n},set:function(a){n=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.bullet=function(){"use str
ict";function b(a,b){var c=a.slice();a.sort(function(a,d){var e=c.indexOf(a),f=c.indexOf(d);return d3.descending(b[e],b[f])})}function c(e){return e.each(function(c,e){var s=p-d.left-d.right,y=q-d.top-d.bottom;r=d3.select(this),a.utils.initSVG(r);var z=g.call(this,c,e).slice(),A=h.call(this,c,e).slice(),B=i.call(this,c,e).slice(),C=j.call(this,c,e).slice(),D=k.call(this,c,e).slice(),E=l.call(this,c,e).slice(),F=m.call(this,c,e).slice(),G=n.call(this,c,e).slice();b(D,z),b(E,A),b(F,B),b(G,C),z.sort(d3.descending),A.sort(d3.descending),B.sort(d3.descending),C.sort(d3.descending);var H=d3.scale.linear().domain(d3.extent(d3.merge([o,z]))).range(f?[s,0]:[0,s]);this.__chart__||d3.scale.linear().domain([0,1/0]).range(H.range());this.__chart__=H;for(var I=(d3.min(z),d3.max(z),z[1],r.selectAll("g.nv-wrap.nv-bullet").data([c])),J=I.enter().append("g").attr("class","nvd3 nv-wrap nv-bullet"),K=J.append("g"),L=I.select("g"),e=0,M=z.length;M>e;e++){var N="nv-range nv-range"+e;2>=e&&(N=N+" nv-range
"+w[e]),K.append("rect").attr("class",N)}K.append("rect").attr("class","nv-measure"),I.attr("transform","translate("+d.left+","+d.top+")");for(var O=function(a){return Math.abs(H(a)-H(0))},P=function(a){return H(0>a?a:0)},e=0,M=z.length;M>e;e++){var Q=z[e];L.select("rect.nv-range"+e).datum(Q).attr("height",y).transition().duration(x).attr("width",O(Q)).attr("x",P(Q))}L.select("rect.nv-measure").style("fill",t).attr("height",y/3).attr("y",y/3).on("mouseover",function(){u.elementMouseover({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).on("mouseout",function(){u.elementMouseout({value:C[0],label:G[0]||"Current",color:d3.select(this).style("fill")})}).transition().duration(x).attr("width",0>C?H(0)-H(C[0]):H(C[0])-H(0)).attr("x",P(C));var R=y/6,S=A.map(function(a,b){return{value:a,label:E[b]}});K.selectAll("path.nv-markerTriangle").data(S).ente
r().append("path").attr("class","nv-markerTriangle").attr("d","M0,"+R+"L"+R+","+-R+" "+-R+","+-R+"Z").on("mouseover",function(a){u.elementMouseover({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill"),pos:[H(a.value),y/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),L.selectAll("path.nv-markerTriangle").data(S).transition().duration(x).attr("transform",function(a){return"translate("+H(a.value)+","+y/2+")"});var T=B.map(function(a,b){return{value:a,label:F[b]}});K.selectAll("line.nv-markerLine").data(T).enter().append("line").attr("cursor","").attr("class","nv-markerLine").attr("x1",function(a){return H(a.value)}).attr("y1","2").attr("x2",function(a){return H(a.value)}).attr("y2",y-2).on("mouseover",function(a){u.elementMouseover({value:a.value,label:
a.label||"Previous",color:d3.select(this).style("fill"),pos:[H(a.value),y/2]})}).on("mousemove",function(a){u.elementMousemove({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){u.elementMouseout({value:a.value,label:a.label||"Previous",color:d3.select(this).style("fill")})}),L.selectAll("line.nv-markerLine").data(T).transition().duration(x).attr("x1",function(a){return H(a.value)}).attr("x2",function(a){return H(a.value)}),I.selectAll(".nv-range").on("mouseover",function(a,b){var c=D[b]||v[b];u.elementMouseover({value:a,label:c,color:d3.select(this).style("fill")})}).on("mousemove",function(){u.elementMousemove({value:C[0],label:G[0]||"Previous",color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){var c=D[b]||v[b];u.elementMouseout({value:a,label:c,color:d3.select(this).style("fill")})})}),c}var d={top:0,right:0,bottom:0,left:0},e="left",f=!1,g=function(a){return a.ranges},h=function(a){return a.markers?a.mark
ers:[]},i=function(a){return a.markerLines?a.markerLines:[0]},j=function(a){return a.measures},k=function(a){return a.rangeLabels?a.rangeLabels:[]},l=function(a){return a.markerLabels?a.markerLabels:[]},m=function(a){return a.markerLineLabels?a.markerLineLabels:[]},n=function(a){return a.measureLabels?a.measureLabels:[]},o=[0],p=380,q=30,r=null,s=null,t=a.utils.getColor(["#1f77b4"]),u=d3.dispatch("elementMouseover","elementMouseout","elementMousemove"),v=["Maximum","Mean","Minimum"],w=["Max","Avg","Min"],x=1e3;return c.dispatch=u,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{ranges:{get:function(){return g},set:function(a){g=a}},markers:{get:function(){return h},set:function(a){h=a}},measures:{get:function(){return j},set:function(a){j=a}},forceX:{get:function(){return o},set:function(a){o=a}},width:{get:function(){return p},set:function(a){p=a}},height:{get:function(){return q},set:function(a){q=a}},tickFormat:{get:function(){return s},set:function(a){s=a}},dur
ation:{get:function(){return x},set:function(a){x=a}},margin:{get:function(){return d},set:function(a){d.top=void 0!==a.top?a.top:d.top,d.right=void 0!==a.right?a.right:d.right,d.bottom=void 0!==a.bottom?a.bottom:d.bottom,d.left=void 0!==a.left?a.left:d.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}},color:{get:function(){return t},set:function(b){t=a.utils.getColor(b)}}}),a.utils.initOptions(c),c},a.models.bulletChart=function(){"use strict";function b(d){return d.each(function(e,o){var p=d3.select(this);a.utils.initSVG(p);var q=a.utils.availableWidth(k,p,g),r=l-g.top-g.bottom;if(b.update=function(){b(d)},b.container=this,!e||!h.call(this,e,o))return a.utils.noData(b,p),b;p.selectAll(".nv-noData").remove();var s=h.call(this,e,o).slice().sort(d3.descending),t=i.call(this,e,o).slice().sort(d3.descending),u=j.call(this,e,o).slice().sort(d3.descending),v=p.selectAll("g.nv-wrap.nv-bulletChart").data([e]),w=v.enter().append("g").attr("class","nvd3
nv-wrap nv-bulletChart"),x=w.append("g"),y=v.select("g");x.append("g").attr("class","nv-bulletWrap"),x.append("g").attr("class","nv-titles"),v.attr("transform","translate("+g.left+","+g.top+")");var z=d3.scale.linear().domain([0,Math.max(s[0],t[0]||0,u[0])]).range(f?[q,0]:[0,q]),A=this.__chart__||d3.scale.linear().domain([0,1/0]).range(z.range());this.__chart__=z;var B=x.select(".nv-titles").append("g").attr("text-anchor","end").attr("transform","translate(-6,"+(l-g.top-g.bottom)/2+")");B.append("text").attr("class","nv-title").text(function(a){return a.title}),B.append("text").attr("class","nv-subtitle").attr("dy","1em").text(function(a){return a.subtitle}),c.width(q).height(r);var C=y.select(".nv-bulletWrap");d3.transition(C).call(c);var D=m||z.tickFormat(q/100),E=y.selectAll("g.nv-tick").data(z.ticks(n?n:q/50),function(a){return this.textContent||D(a)}),F=E.enter().append("g").attr("class","nv-tick").attr("transform",function(a){return"translate("+A(a)+",0)"}).style("opacity",1e-
6);F.append("line").attr("y1",r).attr("y2",7*r/6),F.append("text").attr("text-anchor","middle").attr("dy","1em").attr("y",7*r/6).text(D);var G=d3.transition(E).transition().duration(c.duration()).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1);G.select("line").attr("y1",r).attr("y2",7*r/6),G.select("text").attr("y",7*r/6),d3.transition(E.exit()).transition().duration(c.duration()).attr("transform",function(a){return"translate("+z(a)+",0)"}).style("opacity",1e-6).remove()}),d3.timer.flush(),b}var c=a.models.bullet(),d=a.models.tooltip(),e="left",f=!1,g={top:5,right:40,bottom:20,left:120},h=function(a){return a.ranges},i=function(a){return a.markers?a.markers:[]},j=function(a){return a.measures},k=null,l=55,m=null,n=null,o=null,p=d3.dispatch();return d.duration(0).headerEnabled(!1),c.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:a.label,value:a.value,color:a.color},d.data(a).hidden(!1)}),c.dispatch.on("elementMouseout.tooltip",functio
n(a){d.hidden(!0)}),c.dispatch.on("elementMousemove.tooltip",function(a){d()}),b.bullet=c,b.dispatch=p,b.tooltip=d,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{ranges:{get:function(){return h},set:function(a){h=a}},markers:{get:function(){return i},set:function(a){i=a}},measures:{get:function(){return j},set:function(a){j=a}},width:{get:function(){return k},set:function(a){k=a}},height:{get:function(){return l},set:function(a){l=a}},tickFormat:{get:function(){return m},set:function(a){m=a}},ticks:{get:function(){return n},set:function(a){n=a}},noData:{get:function(){return o},set:function(a){o=a}},margin:{get:function(){return g},set:function(a){g.top=void 0!==a.top?a.top:g.top,g.right=void 0!==a.right?a.right:g.right,g.bottom=void 0!==a.bottom?a.bottom:g.bottom,g.left=void 0!==a.left?a.left:g.left}},orient:{get:function(){return e},set:function(a){e=a,f="right"==e||"bottom"==e}}}),a.utils.inheritOptions(b,c),a.utils.initOptions(b),b},a.models.candlestickBar=fu
nction(){"use strict";function b(x){return x.each(function(b){c=d3.select(this);var x=a.utils.availableWidth(i,c,h),y=a.utils.availableHeight(j,c,h);a.utils.initSVG(c);var A=x/b[0].values.length*.45;l.domain(d||d3.extent(b[0].values.map(n).concat(t))),v?l.range(f||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(f||[5+A/2,x-A/2-5]),m.domain(e||[d3.min(b[0].values.map(s).concat(u)),d3.max(b[0].values.map(r).concat(u))]).range(g||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var B=d3.select(this).selectAll("g.nv-wrap.nv-candlestickBar").data([b[0].values]),C=B.enter().append("g").attr("class","nvd3 nv-wrap nv-candlestickBar"),D=C.append("defs"),E=C.append("g"),F=B.select("g");E.append("g").attr("class","nv-ticks"),B.a
ttr("transform","translate("+h.left+","+h.top+")"),c.on("click",function(a,b){z.chartClick({data:a,index:b,pos:d3.event,id:k})}),D.append("clipPath").attr("id","nv-chart-clip-path-"+k).append("rect"),B.select("#nv-chart-clip-path-"+k+" rect").attr("width",x).attr("height",y),F.attr("clip-path",w?"url(#nv-chart-clip-path-"+k+")":"");var G=B.select(".nv-ticks").selectAll(".nv-tick").data(function(a){return a});G.exit().remove();var H=G.enter().append("g");G.attr("class",function(a,b,c){return(p(a,b)>q(a,b)?"nv-tick negative":"nv-tick positive")+" nv-tick-"+c+"-"+b});H.append("line").attr("class","nv-candlestick-lines").attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),H.append("rect").attr("class","nv-candlestick-rects nv-bars").attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr
("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)});G.select(".nv-candlestick-lines").transition().attr("transform",function(a,b){return"translate("+l(n(a,b))+",0)"}).attr("x1",0).attr("y1",function(a,b){return m(r(a,b))}).attr("x2",0).attr("y2",function(a,b){return m(s(a,b))}),G.select(".nv-candlestick-rects").transition().attr("transform",function(a,b){return"translate("+(l(n(a,b))-A/2)+","+(m(o(a,b))-(p(a,b)>q(a,b)?m(q(a,b))-m(p(a,b)):0))+")"}).attr("x",0).attr("y",0).attr("width",A).attr("height",function(a,b){var c=p(a,b),d=q(a,b);return c>d?m(d)-m(c):m(c)-m(d)})}),b}var c,d,e,f,g,h={top:0,right:0,bottom:0,left:0},i=null,j=null,k=Math.floor(1e4*Math.random()),l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=function(a){return a.y},p=function(a){return a.open},q=function(a){return a.close},r=function(a){return a.high},s=function(a){return a.low},t=[],u=[],v=!1,w=!0,x=a.utils.defaultColor(),y=!1
,z=d3.dispatch("stateChange","changeState","renderEnd","chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove");return b.highlightPoint=function(a,d){b.clearHighlights(),c.select(".nv-candlestickBar .nv-tick-0-"+a).classed("hover",d)},b.clearHighlights=function(){c.select(".nv-candlestickBar .nv-tick.hover").classed("hover",!1)},b.dispatch=z,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return i},set:function(a){i=a}},height:{get:function(){return j},set:function(a){j=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){return m},set:function(a){m=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},forceX:{get:function(){return t},set:function(a){t=a}},forceY:{get:function(){return u},set:functio
n(a){u=a}},padData:{get:function(){return v},set:function(a){v=a}},clipEdge:{get:function(){return w},set:function(a){w=a}},id:{get:function(){return k},set:function(a){k=a}},interactive:{get:function(){return y},set:function(a){y=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},open:{get:function(){return p()},set:function(a){p=a}},close:{get:function(){return q()},set:function(a){q=a}},high:{get:function(){return r},set:function(a){r=a}},low:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return h},set:function(a){h.top=void 0!=a.top?a.top:h.top,h.right=void 0!=a.right?a.right:h.right,h.bottom=void 0!=a.bottom?a.bottom:h.bottom,h.left=void 0!=a.left?a.left:h.left}},color:{get:function(){return x},set:function(b){x=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.cumulativeLineChart=function(){"use strict";function b(l){return I.reset(),I.models(f),s&&I.models(g),t&&I.models(h),l.each(function(l
){function B(a,c){d3.select(b.container).style("cursor","ew-resize")}function F(a,b){H.x=d3.event.x,H.i=Math.round(G.invert(H.x)),L()}function I(a,c){d3.select(b.container).style("cursor","auto"),z.index=H.i,D.stateChange(z)}function L(){ba.data([H]);var a=b.duration();b.duration(0),b.update(),b.duration(a)}var M=d3.select(this);a.utils.initSVG(M),M.classed("nv-chart-"+y,!0);var N=a.utils.availableWidth(p,M,m),O=a.utils.availableHeight(q,M,m);if(b.update=function(){0===E?M.call(b):M.transition().duration(E).call(b)},b.container=this,z.setter(K(l),b.update).getter(J(l)).update(),z.disabled=l.map(function(a){return!!a.disabled}),!A){var P;A={};for(P in z)z[P]instanceof Array?A[P]=z[P].slice(0):A[P]=z[P]}var Q=d3.behavior.drag().on("dragstart",B).on("drag",F).on("dragend",I);if(!(l&&l.length&&l.filter(function(a){return a.values.length}).length))return a.utils.noData(b,M),b;if(M.selectAll(".nv-noData").remove(),d=f.xScale(),e=f.yScale(),x)f.yDomain(null);else{var R=l.filter(function(a)
{return!a.disabled}).map(function(a,b){var c=d3.extent(a.values,f.y());return c[0]<-.95&&(c[0]=-.95),[(c[0]-c[1])/(1+c[1]),(c[1]-c[0])/(1+c[0])]}),S=[d3.min(R,function(a){return a[0]}),d3.max(R,function(a){return a[1]})];f.yDomain(S)}G.domain([0,l[0].values.length-1]).range([0,N]).clamp(!0);var l=c(H.i,l),T=w?"none":"all",U=M.selectAll("g.nv-wrap.nv-cumulativeLine").data([l]),V=U.enter().append("g").attr("class","nvd3 nv-wrap nv-cumulativeLine").append("g"),W=U.select("g");if(V.append("g").attr("class","nv-interactive"),V.append("g").attr("class","nv-x nv-axis").style("pointer-events","none"),V.append("g").attr("class","nv-y nv-axis"),V.append("g").attr("class","nv-background"),V.append("g").attr("class","nv-linesWrap").style("pointer-events",T),V.append("g").attr("class","nv-avgLinesWrap").style("pointer-events","none"),V.append("g").attr("class","nv-legendWrap"),V.append("g").attr("class","nv-controlsWrap"),r?(i.width(N),W.select(".nv-legendWrap").datum(l).call(i),n||i.height()===
m.top||(m.top=i.height(),O=a.utils.availableHeight(q,M,m)),W.select(".nv-legendWrap").attr("transform","translate(0,"+-m.top+")")):W.select(".nv-legendWrap").selectAll("*").remove(),v){var X=[{key:"Re-scale y-axis",disabled:!x}];j.width(140).color(["#444","#444","#444"]).rightAlign(!1).margin({top:5,right:0,bottom:5,left:20}),W.select(".nv-controlsWrap").datum(X).attr("transform","translate(0,"+-m.top+")").call(j)}else W.select(".nv-controlsWrap").selectAll("*").remove();U.attr("transform","translate("+m.left+","+m.top+")"),u&&W.select(".nv-y.nv-axis").attr("transform","translate("+N+",0)");var Y=l.filter(function(a){return a.tempDisabled});U.select(".tempDisabled").remove(),Y.length&&U.append("text").attr("class","tempDisabled").attr("x",N/2).attr("y","-.71em").style("text-anchor","end").text(Y.map(function(a){return a.key}).join(", ")+" values cannot be calculated for this time period."),w&&(k.width(N).height(O).margin({left:m.left,top:m.top}).svgContainer(M).xScale(d),U.select(".
nv-interactive").call(k)),V.select(".nv-background").append("rect"),W.select(".nv-background rect").attr("width",N).attr("height",O),f.y(function(a){return a.display.y}).width(N).height(O).color(l.map(function(a,b){return a.color||o(a,b)}).filter(function(a,b){return!l[b].disabled&&!l[b].tempDisabled}));var Z=W.select(".nv-linesWrap").datum(l.filter(function(a){return!a.disabled&&!a.tempDisabled}));Z.call(f),l.forEach(function(a,b){a.seriesIndex=b});var $=l.filter(function(a){return!a.disabled&&!!C(a)}),_=W.select(".nv-avgLinesWrap").selectAll("line").data($,function(a){return a.key}),aa=function(a){var b=e(C(a));return 0>b?0:b>O?O:b};_.enter().append("line").style("stroke-width",2).style("stroke-dasharray","10,10").style("stroke",function(a,b){return f.color()(a,a.seriesIndex)}).attr("x1",0).attr("x2",N).attr("y1",aa).attr("y2",aa),_.style("stroke-opacity",function(a){var b=e(C(a));return 0>b||b>O?0:1}).attr("x1",0).attr("x2",N).attr("y1",aa).attr("y2",aa),_.exit().remove();var ba=
Z.selectAll(".nv-indexLine").data([H]);ba.enter().append("rect").attr("class","nv-indexLine").attr("width",3).attr("x",-2).attr("fill","red").attr("fill-opacity",.5).style("pointer-events","all").call(Q),ba.attr("transform",function(a){return"translate("+G(a.i)+",0)"}).attr("height",O),s&&(g.scale(d)._ticks(a.utils.calcTicksX(N/70,l)).tickSize(-O,0),W.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),W.select(".nv-x.nv-axis").call(g)),t&&(h.scale(e)._ticks(a.utils.calcTicksY(O/36,l)).tickSize(-N,0),W.select(".nv-y.nv-axis").call(h)),W.select(".nv-background rect").on("click",function(){H.x=d3.mouse(this)[0],H.i=Math.round(G.invert(H.x)),z.index=H.i,D.stateChange(z),L()}),f.dispatch.on("elementClick",function(a){H.i=a.pointIndex,H.x=G(H.i),z.index=H.i,D.stateChange(z),L()}),j.dispatch.on("legendClick",function(a,c){a.disabled=!a.disabled,x=!a.disabled,z.rescaleY=x,D.stateChange(z),b.update()}),i.dispatch.on("stateChange",function(a){for(var c in a)z[c]=a[c];D.
stateChange(z),b.update()}),k.dispatch.on("elementMousemove",function(c){f.clearHighlights();var d,e,i,j=[];if(l.filter(function(a,b){return a.seriesIndex=b,!a.disabled}).forEach(function(g,h){e=a.interactiveBisect(g.values,c.pointXValue,b.x()),f.highlightPoint(h,e,!0);var k=g.values[e];"undefined"!=typeof k&&("undefined"==typeof d&&(d=k),"undefined"==typeof i&&(i=b.xScale()(b.x()(k,e))),j.push({key:g.key,value:b.y()(k,e),color:o(g,g.seriesIndex)}))}),j.length>2){var m=b.yScale().invert(c.mouseY),n=Math.abs(b.yScale().domain()[0]-b.yScale().domain()[1]),p=.03*n,q=a.nearestValueIndex(j.map(function(a){return a.value}),m,p);null!==q&&(j[q].highlight=!0)}var r=g.tickFormat()(b.x()(d,e),e);k.tooltip.valueFormatter(function(a,b){return h.tickFormat()(a)}).data({value:r,series:j})(),k.renderGuideLine(i)}),k.dispatch.on("elementMouseout",function(a){f.clearHighlights()}),D.on("changeState",function(a){"undefined"!=typeof a.disabled&&(l.forEach(function(b,c){b.disabled=a.disabled[c]}),z.dis
abled=a.disabled),"undefined"!=typeof a.index&&(H.i=a.index,H.x=G(H.i),z.index=a.index,ba.data([H])),"undefined"!=typeof a.rescaleY&&(x=a.rescaleY),b.update()})}),I.renderEnd("cumulativeLineChart immediate"),b}function c(a,b){return L||(L=f.y()),b.map(function(b,c){if(!b.values)return b;var d=b.values[a];if(null==d)return b;var e=L(d,a);return-.95>e&&!F?(b.tempDisabled=!0,b):(b.tempDisabled=!1,b.values=b.values.map(function(a,b){return a.display={y:(L(a,b)-e)/(1+e)},a}),b)})}var d,e,f=a.models.line(),g=a.models.axis(),h=a.models.axis(),i=a.models.legend(),j=a.models.legend(),k=a.interactiveGuideline(),l=a.models.tooltip(),m={top:30,right:30,bottom:50,left:60},n=null,o=a.utils.defaultColor(),p=null,q=null,r=!0,s=!0,t=!0,u=!1,v=!0,w=!1,x=!0,y=f.id(),z=a.utils.state(),A=null,B=null,C=function(a){return a.average},D=d3.dispatch("stateChange","changeState","renderEnd"),E=250,F=!1;z.index=0,z.rescaleY=x,g.orient("bottom").tickPadding(7),h.orient(u?"right":"left"),l.valueFormatter(function
(a,b){return h.tickFormat()(a,b)}).headerFormatter(function(a,b){return g.tickFormat()(a,b)}),j.updateState(!1);var G=d3.scale.linear(),H={i:0,x:0},I=a.utils.renderWatch(D,E),J=function(a){return function(){return{active:a.map(function(a){return!a.disabled}),index:H.i,rescaleY:x}}},K=function(a){return function(b){void 0!==b.index&&(H.i=b.index),void 0!==b.rescaleY&&(x=b.rescaleY),void 0!==b.active&&a.forEach(function(a,c){a.disabled=!b.active[c]})}};f.dispatch.on("elementMouseover.tooltip",function(a){var c={x:b.x()(a.point),y:b.y()(a.point),color:a.point.color};a.point=c,l.data(a).hidden(!1)}),f.dispatch.on("elementMouseout.tooltip",function(a){l.hidden(!0)});var L=null;return b.dispatch=D,b.lines=f,b.legend=i,b.controls=j,b.xAxis=g,b.yAxis=h,b.interactiveLayer=k,b.state=z,b.tooltip=l,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return p},set:function(a){p=a}},height:{get:function(){return q},set:function(a){q=a}},rescaleY:{get:function(
){return x},set:function(a){x=a}},showControls:{get:function(){return v},set:function(a){v=a}},showLegend:{get:function(){return r},set:function(a){r=a}},average:{get:function(){return C},set:function(a){C=a}},defaultState:{get:function(){return A},set:function(a){A=a}},noData:{get:function(){return B},set:function(a){B=a}},showXAxis:{get:function(){return s},set:function(a){s=a}},showYAxis:{get:function(){return t},set:function(a){t=a}},noErrorCheck:{get:function(){return F},set:function(a){F=a}},margin:{get:function(){return m},set:function(a){void 0!==a.top&&(m.top=a.top,n=a.top),m.right=void 0!==a.right?a.right:m.right,m.bottom=void 0!==a.bottom?a.bottom:m.bottom,m.left=void 0!==a.left?a.left:m.left}},color:{get:function(){return o},set:function(b){o=a.utils.getColor(b),i.color(o)}},useInteractiveGuideline:{get:function(){return w},set:function(a){w=a,a===!0&&(b.interactive(!1),b.useVoronoi(!1))}},rightAlignYAxis:{get:function(){return u},set:function(a){u=a,h.orient(a?"right":"
left")}},duration:{get:function(){return E},set:function(a){E=a,f.duration(E),g.duration(E),h.duration(E),I.reset(E)}}}),a.utils.inheritOptions(b,f),a.utils.initOptions(b),b},a.models.discreteBar=function(){"use strict";function b(m){return y.reset(),m.each(function(b){var m=k-j.left-j.right,x=l-j.top-j.bottom;c=d3.select(this),a.utils.initSVG(c),b.forEach(function(a,b){a.values.forEach(function(a){a.series=b})});var z=d&&e?[]:b.map(function(a){return a.values.map(function(a,b){return{x:p(a,b),y:q(a,b),y0:a.y0}})});n.domain(d||d3.merge(z).map(function(a){return a.x})).rangeBands(f||[0,m],.1),o.domain(e||d3.extent(d3.merge(z).map(function(a){return a.y}).concat(r))),t?o.range(g||[x-(o.domain()[0]<0?12:0),o.domain()[1]>0?12:0]):o.range(g||[x,0]),h=h||n,i=i||o.copy().range([o(0),o(0)]);var A=c.selectAll("g.nv-wrap.nv-discretebar").data([b]),B=A.enter().append("g").attr("class","nvd3 nv-wrap nv-discretebar"),C=B.append("g");A.select("g");C.append("g").attr("class","nv-groups"),A.attr("t
ransform","translate("+j.left+","+j.top+")");var D=A.select(".nv-groups").selectAll(".nv-group").data(function(a){return a},function(a){return a.key});D.enter().append("g").style("stroke-opacity",1e-6).style("fill-opacity",1e-6),D.exit().watchTransition(y,"discreteBar: exit groups").style("stroke-opacity",1e-6).style("fill-opacity",1e-6).remove(),D.attr("class",function(a,b){return"nv-group nv-series-"+b}).classed("hover",function(a){return a.hover}),D.watchTransition(y,"discreteBar: groups").style("stroke-opacity",1).style("fill-opacity",.75);var E=D.selectAll("g.nv-bar").data(function(a){return a.values});E.exit().remove();var F=E.enter().append("g").attr("transform",function(a,b,c){return"translate("+(n(p(a,b))+.05*n.rangeBand())+", "+o(0)+")"}).on("mouseover",function(a,b){d3.select(this).classed("hover",!0),v.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")})}).on("mouseout",function(a,b){d3.select(this).classed("hover",!1),v.elementMouseout({data:a,index:b,
color:d3.select(this).style("fill")})}).on("mousemove",function(a,b){v.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){var c=this;v.elementClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}).on("dblclick",function(a,b){v.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation()});F.append("rect").attr("height",0).attr("width",.9*n.rangeBand()/b.length),t?(F.append("text").attr("text-anchor","middle"),E.select("text").text(function(a,b){return u(q(a,b))}).watchTransition(y,"discreteBar: bars text").attr("x",.9*n.rangeBand()/2).attr("y",function(a,b){return q(a,b)<0?o(q(a,b))-o(0)+12:-4})):E.selectAll("text").remove(),E.attr("class",function(a,b){return q(a,b)<0?"nv-bar negative":"nv-bar positive"}).style("fill",function(a,b){return a.color||s(a,b)}).style("stroke",function(a,b){return a.color||s(a,b)}).select("rect").attr("class",w).wa
tchTransition(y,"discreteBar: bars rect").attr("width",.9*n.rangeBand()/b.length),E.watchTransition(y,"discreteBar: bars").attr("transform",function(a,b){var c=n(p(a,b))+.05*n.rangeBand(),d=q(a,b)<0?o(0):o(0)-o(q(a,b))<1?o(0)-1:o(q(a,b));return"translate("+c+", "+d+")"}).select("rect").attr("height",function(a,b){return Math.max(Math.abs(o(q(a,b))-o(0)),1)}),h=n.copy(),i=o.copy()}),y.renderEnd("discreteBar immediate"),b}var c,d,e,f,g,h,i,j={top:0,right:0,bottom:0,left:0},k=960,l=500,m=Math.floor(1e4*Math.random()),n=d3.scale.ordinal(),o=d3.scale.linear(),p=function(a){return a.x},q=function(a){return a.y},r=[0],s=a.utils.defaultColor(),t=!1,u=d3.format(",.2f"),v=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),w="discreteBar",x=250,y=a.utils.renderWatch(v,x);return b.dispatch=v,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},height:{g
et:function(){return l},set:function(a){l=a}},forceY:{get:function(){return r},set:function(a){r=a;
-}},showValues:{get:function(){return t},set:function(a){t=a}},x:{get:function(){return p},set:function(a){p=a}},y:{get:function(){return q},set:function(a){q=a}},xScale:{get:function(){return n},set:function(a){n=a}},yScale:{get:function(){return o},set:function(a){o=a}},xDomain:{get:function(){return d},set:function(a){d=a}},yDomain:{get:function(){return e},set:function(a){e=a}},xRange:{get:function(){return f},set:function(a){f=a}},yRange:{get:function(){return g},set:function(a){g=a}},valueFormat:{get:function(){return u},set:function(a){u=a}},id:{get:function(){return m},set:function(a){m=a}},rectClass:{get:function(){return w},set:function(a){w=a}},margin:{get:function(){return j},set:function(a){j.top=void 0!==a.top?a.top:j.top,j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},color:{get:function(){return s},set:function(b){s=a.utils.getColor(b)}},duration:{get:function(){return x},set:function(a){x=a,
y.reset(x)}}}),a.utils.initOptions(b),b},a.models.discreteBarChart=function(){"use strict";function b(i){return y.reset(),y.models(e),p&&y.models(f),q&&y.models(g),i.each(function(i){var n=d3.select(this);a.utils.initSVG(n);var v=a.utils.availableWidth(l,n,j),y=a.utils.availableHeight(m,n,j);if(b.update=function(){w.beforeUpdate(),n.transition().duration(x).call(b)},b.container=this,!(i&&i.length&&i.filter(function(a){return a.values.length}).length))return a.utils.noData(b,n),b;n.selectAll(".nv-noData").remove(),c=e.xScale(),d=e.yScale().clamp(!0);var z=n.selectAll("g.nv-wrap.nv-discreteBarWithAxes").data([i]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-discreteBarWithAxes").append("g"),B=A.append("defs"),C=z.select("g");A.append("g").attr("class","nv-x nv-axis"),A.append("g").attr("class","nv-y nv-axis").append("g").attr("class","nv-zeroLine").append("line"),A.append("g").attr("class","nv-barsWrap"),A.append("g").attr("class","nv-legendWrap"),C.attr("transform","translat
e("+j.left+","+j.top+")"),o?(h.width(v),C.select(".nv-legendWrap").datum(i).call(h),k||h.height()===j.top||(j.top=h.height(),y=a.utils.availableHeight(m,n,j)),z.select(".nv-legendWrap").attr("transform","translate(0,"+-j.top+")")):C.select(".nv-legendWrap").selectAll("*").remove(),r&&C.select(".nv-y.nv-axis").attr("transform","translate("+v+",0)"),e.width(v).height(y);var D=C.select(".nv-barsWrap").datum(i.filter(function(a){return!a.disabled}));if(D.transition().call(e),B.append("clipPath").attr("id","nv-x-label-clip-"+e.id()).append("rect"),C.select("#nv-x-label-clip-"+e.id()+" rect").attr("width",c.rangeBand()*(s?2:1)).attr("height",16).attr("x",-c.rangeBand()/(s?1:2)),p){f.scale(c)._ticks(a.utils.calcTicksX(v/100,i)).tickSize(-y,0),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+(d.range()[0]+(e.showValues()&&d.domain()[0]<0?16:0))+")"),C.select(".nv-x.nv-axis").call(f);var E=C.select(".nv-x.nv-axis").selectAll("g");s&&E.selectAll("text").attr("transform",function(a,b,
c){return"translate(0,"+(c%2==0?"5":"17")+")"}),u&&E.selectAll(".tick text").attr("transform","rotate("+u+" 0,0)").style("text-anchor",u>0?"start":"end"),t&&C.selectAll(".tick text").call(a.utils.wrapTicks,b.xAxis.rangeBand())}q&&(g.scale(d)._ticks(a.utils.calcTicksY(y/36,i)).tickSize(-v,0),C.select(".nv-y.nv-axis").call(g)),C.select(".nv-zeroLine line").attr("x1",0).attr("x2",r?-v:v).attr("y1",d(0)).attr("y2",d(0))}),y.renderEnd("discreteBar chart immediate"),b}var c,d,e=a.models.discreteBar(),f=a.models.axis(),g=a.models.axis(),h=a.models.legend(),i=a.models.tooltip(),j={top:15,right:10,bottom:50,left:60},k=null,l=null,m=null,n=a.utils.getColor(),o=!1,p=!0,q=!0,r=!1,s=!1,t=!1,u=0,v=null,w=d3.dispatch("beforeUpdate","renderEnd"),x=250;f.orient("bottom").showMaxMin(!1).tickFormat(function(a){return a}),g.orient(r?"right":"left").tickFormat(d3.format(",.1f")),i.duration(0).headerEnabled(!1).valueFormatter(function(a,b){return g.tickFormat()(a,b)}).keyFormatter(function(a,b){return f.
tickFormat()(a,b)});var y=a.utils.renderWatch(w,x);return e.dispatch.on("elementMouseover.tooltip",function(a){a.series={key:b.x()(a.data),value:b.y()(a.data),color:a.color},i.data(a).hidden(!1)}),e.dispatch.on("elementMouseout.tooltip",function(a){i.hidden(!0)}),e.dispatch.on("elementMousemove.tooltip",function(a){i()}),b.dispatch=w,b.discretebar=e,b.legend=h,b.xAxis=f,b.yAxis=g,b.tooltip=i,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return l},set:function(a){l=a}},height:{get:function(){return m},set:function(a){m=a}},showLegend:{get:function(){return o},set:function(a){o=a}},staggerLabels:{get:function(){return s},set:function(a){s=a}},rotateLabels:{get:function(){return u},set:function(a){u=a}},wrapLabels:{get:function(){return t},set:function(a){t=!!a}},showXAxis:{get:function(){return p},set:function(a){p=a}},showYAxis:{get:function(){return q},set:function(a){q=a}},noData:{get:function(){return v},set:function(a){v=a}},margin:{get:
function(){return j},set:function(a){void 0!==a.top&&(j.top=a.top,k=a.top),j.right=void 0!==a.right?a.right:j.right,j.bottom=void 0!==a.bottom?a.bottom:j.bottom,j.left=void 0!==a.left?a.left:j.left}},duration:{get:function(){return x},set:function(a){x=a,y.reset(x),e.duration(x),f.duration(x),g.duration(x)}},color:{get:function(){return n},set:function(b){n=a.utils.getColor(b),e.color(n),h.color(n)}},rightAlignYAxis:{get:function(){return r},set:function(a){r=a,g.orient(a?"right":"left")}}}),a.utils.inheritOptions(b,e),a.utils.initOptions(b),b},a.models.distribution=function(){"use strict";function b(k){return m.reset(),k.each(function(b){var k=(e-("x"===g?d.left+d.right:d.top+d.bottom),"x"==g?"y":"x"),l=d3.select(this);a.utils.initSVG(l),c=c||j;var n=l.selectAll("g.nv-distribution").data([b]),o=n.enter().append("g").attr("class","nvd3 nv-distribution"),p=(o.append("g"),n.select("g"));n.attr("transform","translate("+d.left+","+d.top+")");var q=p.selectAll("g.nv-dist").data(function(
a){return a},function(a){return a.key});q.enter().append("g"),q.attr("class",function(a,b){return"nv-dist nv-series-"+b}).style("stroke",function(a,b){return i(a,b)});var r=q.selectAll("line.nv-dist"+g).data(function(a){return a.values});r.enter().append("line").attr(g+"1",function(a,b){return c(h(a,b))}).attr(g+"2",function(a,b){return c(h(a,b))}),m.transition(q.exit().selectAll("line.nv-dist"+g),"dist exit").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}).style("stroke-opacity",0).remove(),r.attr("class",function(a,b){return"nv-dist"+g+" nv-dist"+g+"-"+b}).attr(k+"1",0).attr(k+"2",f),m.transition(r,"dist").attr(g+"1",function(a,b){return j(h(a,b))}).attr(g+"2",function(a,b){return j(h(a,b))}),c=j.copy()}),m.renderEnd("distribution immediate"),b}var c,d={top:0,right:0,bottom:0,left:0},e=400,f=8,g="x",h=function(a){return a[g]},i=a.utils.defaultColor(),j=d3.scale.linear(),k=250,l=d3.dispatch("renderEnd"),m=a.utils.renderWatch(l,k);return b.opt
ions=a.utils.optionsFunc.bind(b),b.dispatch=l,b.margin=function(a){return arguments.length?(d.top="undefined"!=typeof a.top?a.top:d.top,d.right="undefined"!=typeof a.right?a.right:d.right,d.bottom="undefined"!=typeof a.bottom?a.bottom:d.bottom,d.left="undefined"!=typeof a.left?a.left:d.left,b):d},b.width=function(a){return arguments.length?(e=a,b):e},b.axis=function(a){return arguments.length?(g=a,b):g},b.size=function(a){return arguments.length?(f=a,b):f},b.getData=function(a){return arguments.length?(h=d3.functor(a),b):h},b.scale=function(a){return arguments.length?(j=a,b):j},b.color=function(c){return arguments.length?(i=a.utils.getColor(c),b):i},b.duration=function(a){return arguments.length?(k=a,m.reset(k),b):k},b},a.models.focus=function(b){"use strict";function c(u){return t.reset(),t.models(b),m&&t.models(f),n&&t.models(g),u.each(function(t){function u(a){var b=+("e"==a),c=b?1:-1,d=z/3;return"M"+.5*c+","+d+"A6,6 0 0 "+b+" "+6.5*c+","+(d+6)+"V"+(2*d-6)+"A6,6 0 0 "+b+" "+.5*c+
","+2*d+"ZM"+2.5*c+","+(d+8)+"V"+(2*d-8)+"M"+4.5*c+","+(d+8)+"V"+(2*d-8)}function v(){h.empty()||h.extent(p),E.data([h.empty()?d.domain():p]).each(function(a,b){var c=d(a[0])-d.range()[0],e=y-d(a[1]);d3.select(this).select(".left").attr("width",0>c?0:c),d3.select(this).select(".right").attr("x",d(a[1])).attr("width",0>e?0:e)})}function w(a){p=h.empty()?null:h.extent();var b=h.empty()?d.domain():h.extent();r.brush({extent:b,brush:h}),v(),a&&r.onBrush(b)}var x=d3.select(this);a.utils.initSVG(x);var y=a.utils.availableWidth(k,x,i),z=l-i.top-i.bottom;c.update=function(){0===q?x.call(c):x.transition().duration(q).call(c)},c.container=this,d=b.xScale(),e=b.yScale();var A=x.selectAll("g.nv-focus").data([t]),B=A.enter().append("g").attr("class","nvd3 nv-focus").append("g"),C=A.select("g");A.attr("transform","translate("+i.left+","+i.top+")"),B.append("g").attr("class","nv-background").append("rect"),B.append("g").attr("class","nv-x nv-axis"),B.append("g").attr("class","nv-y nv-axis"),B.appe
nd("g").attr("class","nv-contentWrap"),B.append("g").attr("class","nv-brushBackground"),B.append("g").attr("class","nv-x nv-brush"),o&&C.select(".nv-y.nv-axis").attr("transform","translate("+y+",0)"),C.select(".nv-background rect").attr("width",y).attr("height",z),b.width(y).height(z).color(t.map(function(a,b){return a.color||j(a,b)}).filter(function(a,b){return!t[b].disabled}));var D=C.select(".nv-contentWrap").datum(t.filter(function(a){return!a.disabled}));d3.transition(D).call(b),h.x(d).on("brush",function(){w(s)}),h.on("brushend",function(){s||r.onBrush(h.empty()?d.domain():h.extent())}),p&&h.extent(p);var E=C.select(".nv-brushBackground").selectAll("g").data([p||h.extent()]),F=E.enter().append("g");F.append("rect").attr("class","left").attr("x",0).attr("y",0).attr("height",z),F.append("rect").attr("class","right").attr("x",0).attr("y",0).attr("height",z);var G=C.select(".nv-x.nv-brush").call(h);G.selectAll("rect").attr("height",z),G.selectAll(".resize").append("path").attr("d"
,u),w(!0),C.select(".nv-background rect").attr("width",y).attr("height",z),m&&(f.scale(d)._ticks(a.utils.calcTicksX(y/100,t)).tickSize(-z,0),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")"),d3.transition(C.select(".nv-x.nv-axis")).call(f)),n&&(g.scale(e)._ticks(a.utils.calcTicksY(z/36,t)).tickSize(-y,0),d3.transition(C.select(".nv-y.nv-axis")).call(g)),C.select(".nv-x.nv-axis").attr("transform","translate(0,"+e.range()[0]+")")}),t.renderEnd("focus immediate"),c}var d,e,b=b||a.models.line(),f=a.models.axis(),g=a.models.axis(),h=d3.svg.brush(),i={top:10,right:0,bottom:30,left:0},j=a.utils.defaultColor(),k=null,l=70,m=!0,n=!1,o=!1,p=null,q=250,r=d3.dispatch("brush","onBrush","renderEnd"),s=!0;b.interactive(!1),b.pointActive(function(a){return!1});var t=a.utils.renderWatch(r,q);return c.dispatch=r,c.content=b,c.brush=h,c.xAxis=f,c.yAxis=g,c.options=a.utils.optionsFunc.bind(c),c._options=Object.create({},{width:{get:function(){return k},set:function(a){k=a}},he
ight:{get:function(){return l},set:function(a){l=a}},showXAxis:{get:function(){return m},set:function(a){m=a}},showYAxis:{get:function(){return n},set:function(a){n=a}},brushExtent:{get:function(){return p},set:function(a){p=a}},syncBrushing:{get:function(){return s},set:function(a){s=a}},margin:{get:function(){return i},set:function(a){i.top=void 0!==a.top?a.top:i.top,i.right=void 0!==a.right?a.right:i.right,i.bottom=void 0!==a.bottom?a.bottom:i.bottom,i.left=void 0!==a.left?a.left:i.left}},duration:{get:function(){return q},set:function(a){q=a,t.reset(q),b.duration(q),f.duration(q),g.duration(q)}},color:{get:function(){return j},set:function(c){j=a.utils.getColor(c),b.color(j)}},interpolate:{get:function(){return b.interpolate()},set:function(a){b.interpolate(a)}},xTickFormat:{get:function(){return f.tickFormat()},set:function(a){f.tickFormat(a)}},yTickFormat:{get:function(){return g.tickFormat()},set:function(a){g.tickFormat(a)}},x:{get:function(){return b.x()},set:function(a){b.
x(a)}},y:{get:function(){return b.y()},set:function(a){b.y(a)}},rightAlignYAxis:{get:function(){return o},set:function(a){o=a,g.orient(o?"right":"left")}}}),a.utils.inheritOptions(c,b),a.utils.initOptions(c),c},a.models.forceDirectedGraph=function(){"use strict";function b(g){return u.reset(),g.each(function(g){f=d3.select(this),a.utils.initSVG(f);var j=a.utils.availableWidth(d,f,c),u=a.utils.availableHeight(e,f,c);if(f.attr("width",j).attr("height",u),!(g&&g.links&&g.nodes))return a.utils.noData(b,f),b;f.selectAll(".nv-noData").remove(),f.selectAll("*").remove();var v=new Set;g.nodes.forEach(function(a){var b=Object.keys(a);b.forEach(function(a){v.add(a)})});var w=d3.layout.force().nodes(g.nodes).links(g.links).size([j,u]).linkStrength(k).friction(l).linkDistance(m).charge(n).gravity(o).theta(p).alpha(q).start(),x=f.selectAll(".link").data(g.links).enter().append("line").attr("class","nv-force-link").style("stroke-width",function(a){return Math.sqrt(a.value)}),y=f.selectAll(".node"
).data(g.nodes).enter().append("g").attr("class","nv-force-node").call(w.drag);y.append("circle").attr("r",r).style("fill",function(a){return h(a)}).on("mouseover",function(a){f.select(".nv-series-"+a.seriesIndex+" .nv-distx-"+a.pointIndex).attr("y1",a.py),f.select(".nv-series-"+a.seriesIndex+" .nv-disty-"+a.pointIndex).attr("x2",a.px);var b=h(a);a.series=[],v.forEach(function(c){a.series.push({color:b,key:c,value:a[c]})}),i.data(a).hidden(!1)}).on("mouseout",function(a){i.hidden(!0)}),i.headerFormatter(function(a){return"Node"}),t(x),s(y),w.on("tick",function(){x.attr("x1",function(a){return a.source.x}).attr("y1",function(a){return a.source.y}).attr("x2",function(a){return a.target.x}).attr("y2",function(a){return a.target.y}),y.attr("transform",function(a){return"translate("+a.x+", "+a.y+")"})})}),b}var c={top:2,right:0,bottom:2,left:0},d=400,e=32,f=null,g=d3.dispatch("renderEnd"),h=a.utils.getColor(["#000"]),i=a.models.tooltip(),j=null,k=.1,l=.9,m=30,n=-120,o=.1,p=.8,q=.1,r=5,s=
function(a){},t=function(a){},u=a.utils.renderWatch(g);return b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},linkStrength:{get:function(){return k},set:function(a){k=a}},friction:{get:function(){return l},set:function(a){l=a}},linkDist:{get:function(){return m},set:function(a){m=a}},charge:{get:function(){return n},set:function(a){n=a}},gravity:{get:function(){return o},set:function(a){o=a}},theta:{get:function(){return p},set:function(a){p=a}},alpha:{get:function(){return q},set:function(a){q=a}},radius:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return getX},set:function(a){getX=d3.functor(a)}},y:{get:function(){return getY},set:function(a){getY=d3.functor(a)}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left
=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}},noData:{get:function(){return j},set:function(a){j=a}},nodeExtras:{get:function(){return s},set:function(a){s=a}},linkExtras:{get:function(){return t},set:function(a){t=a}}}),b.dispatch=g,b.tooltip=i,a.utils.initOptions(b),b},a.models.furiousLegend=function(){"use strict";function b(r){function s(a,b){return"furious"!=q?"#000":o?a.disengaged?h(a,b):"#fff":o?void 0:a.disabled?h(a,b):"#fff"}function t(a,b){return o&&"furious"==q?a.disengaged?"#fff":h(a,b):a.disabled?"#fff":h(a,b)}return r.each(function(b){var r=d-c.left-c.right,u=d3.select(this);a.utils.initSVG(u);var v=u.selectAll("g.nv-legend").data([b]),w=(v.enter().append("g").attr("class","nvd3 nv-legend").append("g"),v.select("g"));v.attr("transform","translate("+c.left+","+c.top+")");var x,y=w.selectAll(".nv-series").data(function(a){return"furious"!=q?a:a.filter(function(a){return o?!0:!a.disengaged})}),z=y.enter().append("
g").attr("class","nv-series");if("classic"==q)z.append("circle").style("stroke-width",2).attr("class","nv-legend-symbol").attr("r",5),x=y.select("circle");else if("furious"==q){z.append("rect").style("stroke-width",2).attr("class","nv-legend-symbol").attr("rx",3).attr("ry",3),x=y.select("rect"),z.append("g").attr("class","nv-check-box").property("innerHTML",'<path d="M0.5,5 L22.5,5 L22.5,26.5 L0.5,26.5 L0.5,5 Z" class="nv-box"></path><path d="M5.5,12.8618467 L11.9185089,19.2803556 L31,0.198864511" class="nv-check"></path>').attr("transform","translate(-10,-8)scale(0.5)");var A=y.select(".nv-check-box");A.each(function(a,b){d3.select(this).selectAll("path").attr("stroke",s(a,b))})}z.append("text").attr("text-anchor","start").attr("class","nv-legend-text").attr("dy",".32em").attr("dx","8");var B=y.select("text.nv-legend-text");y.on("mouseover",function(a,b){p.legendMouseover(a,b)}).on("mouseout",function(a,b){p.legendMouseout(a,b)}).on("click",function(a,b){p.legendClick(a,b);var c=y.
data();if(m){if("classic"==q)n?(c.forEach(function(a){a.disabled=!0}),a.disabled=!1):(a.disabled=!a.disabled,c.every(function(a){return a.disabled})&&c.forEach(function(a){a.disabled=!1}));else if("furious"==q)if(o)a.disengaged=!a.disengaged,a.userDisabled=void 0==a.userDisabled?!!a.disabled:a.userDisabled,a.disabled=a.disengaged||a.userDisabled;else if(!o){a.disabled=!a.disabled,a.userDisabled=a.disabled;var d=c.filter(function(a){return!a.disengaged});d.every(function(a){return a.userDisabled})&&c.forEach(function(a){a.disabled=a.userDisabled=!1})}p.stateChange({disabled:c.map(function(a){return!!a.disabled}),disengaged:c.map(function(a){return!!a.disengaged})})}}).on("dblclick",function(a,b){if(("furious"!=q||!o)&&(p.legendDblclick(a,b),m)){var c=y.data();c.forEach(function(a){a.disabled=!0,"furious"==q&&(a.userDisabled=a.disabled)}),a.disabled=!1,"furious"==q&&(a.userDisabled=a.disabled),p.stateChange({disabled:c.map(function(a){return!!a.disabled})})}}),y.classed("nv-disabled",
function(a){return a.userDisabled}),y.exit().remove(),B.attr("fill",s).text(function(a){return g(f(a))});var C;switch(q){case"furious":C=23;break;case"classic":C=20}if(j){var D=[];y.each(function(b,c){var d;if(g(f(b))&&g(f(b)).length>i){var e=g(f(b)).substring(0,i);d=d3.select(this).select("text").text(e+"..."),d3.select(this).append("svg:title").text(g(f(b)))}else d=d3.select(this).select("text");var h;try{if(h=d.node().getComputedTextLength(),0>=h)throw Error()}catch(j){h=a.utils.calcApproxTextWidth(d)}D.push(h+k)});for(var E=0,F=0,G=[];r>F&&E<D.length;)G[E]=D[E],F+=D[E++];for(0===E&&(E=1);F>r&&E>1;){G=[],E--;for(var H=0;H<D.length;H++)D[H]>(G[H%E]||0)&&(G[H%E]=D[H]);F=G.reduce(function(a,b,c,d){return a+b})}for(var I=[],J=0,K=0;E>J;J++)I[J]=K,K+=G[J];y.attr("transform",function(a,b){return"translate("+I[b%E]+","+(5+Math.floor(b/E)*C)+")"}),l?w.attr("transform","translate("+(d-c.right-F)+","+c.top+")"):w.attr("transform","translate(0,"+c.top+")"),e=c.top+c.bottom+Math.ceil(D.lengt
h/E)*C}else{var L,M=5,N=5,O=0;y.attr("transform",function(a,b){var e=d3.select(this).select("text").node().getComputedTextLength()+k;return L=N,d<c.left+c.right+L+e&&(N=L=5,M+=C),N+=e,N>O&&(O=N),"translate("+L+","+M+")"}),w.attr("transform","translate("+(d-c.right-O)+","+c.top+")"),e=c.top+c.bottom+M+15}"furious"==q&&x.attr("width",function(a,b){return B[0][b].getComputedTextLength()+27}).attr("height",18).attr("y",-9).attr("x",-15),x.style("fill",t).style("stroke",function(a,b){return a.color||h(a,b)})}),b}var c={top:5,right:0,bottom:5,left:0},d=400,e=20,f=function(a){return a.key},g=function(a){return a},h=a.utils.getColor(),i=20,j=!0,k=28,l=!0,m=!0,n=!1,o=!1,p=d3.dispatch("legendClick","legendDblclick","legendMouseover","legendMouseout","stateChange"),q="classic";return b.dispatch=p,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return d},set:function(a){d=a}},height:{get:function(){return e},set:function(a){e=a}},key:{get:function(){retu
rn f},set:function(a){f=a}},keyFormatter:{get:function(){return g},set:function(a){g=a}},align:{get:function(){return j},set:function(a){j=a}},rightAlign:{get:function(){return l},set:function(a){l=a}},maxKeyLength:{get:function(){return i},set:function(a){i=a}},padding:{get:function(){return k},set:function(a){k=a}},updateState:{get:function(){return m},set:function(a){m=a}},radioButtonMode:{get:function(){return n},set:function(a){n=a}},expanded:{get:function(){return o},set:function(a){o=a}},vers:{get:function(){return q},set:function(a){q=a}},margin:{get:function(){return c},set:function(a){c.top=void 0!==a.top?a.top:c.top,c.right=void 0!==a.right?a.right:c.right,c.bottom=void 0!==a.bottom?a.bottom:c.bottom,c.left=void 0!==a.left?a.left:c.left}},color:{get:function(){return h},set:function(b){h=a.utils.getColor(b)}}}),a.utils.initOptions(b),b},a.models.historicalBar=function(){"use strict";function b(x){return x.each(function(b){w.reset(),k=d3.select(this);var x=a.utils.availabl
eWidth(h,k,g),y=a.utils.availableHeight(i,k,g);a.utils.initSVG(k),l.domain(c||d3.extent(b[0].values.map(n).concat(p))),r?l.range(e||[.5*x/b[0].values.length,x*(b[0].values.length-.5)/b[0].values.length]):l.range(e||[0,x]),m.domain(d||d3.extent(b[0].values.map(o).concat(q))).range(f||[y,0]),l.domain()[0]===l.domain()[1]&&(l.domain()[0]?l.domain([l.domain()[0]-.01*l.domain()[0],l.domain()[1]+.01*l.domain()[1]]):l.domain([-1,1])),m.domain()[0]===m.domain()[1]&&(m.domain()[0]?m.domain([m.domain()[0]+.01*m.domain()[0],m.domain()[1]-.01*m.domain()[1]]):m.domain([-1,1]));var z=k.selectAll("g.nv-wrap.nv-historicalBar-"+j).data([b[0].values]),A=z.enter().append("g").attr("class","nvd3 nv-wrap nv-historicalBar-"+j),B=A.append("defs"),C=A.append("g"),D=z.select("g");C.append("g").attr("class","nv-bars"),z.attr("transform","translate("+g.left+","+g.top+")"),k.on("click",function(a,b){u.chartClick({data:a,index:b,pos:d3.event,id:j})}),B.append("clipPath").attr("id","nv-chart-clip-path-"+j).appen
d("rect"),z.select("#nv-chart-clip-path-"+j+" rect").attr("width",x).attr("height",y),D.attr("clip-path",s?"url(#nv-chart-clip-path-"+j+")":"");var E=z.select(".nv-bars").selectAll(".nv-bar").data(function(a){return a},function(a,b){return n(a,b)});E.exit().remove(),E.enter().append("rect").attr("x",0).attr("y",function(b,c){return a.utils.NaNtoZero(m(Math.max(0,o(b,c))))}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.abs(m(o(b,c))-m(0)))}).attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).on("mouseover",function(a,b){v&&(d3.select(this).classed("hover",!0),u.elementMouseover({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mouseout",function(a,b){v&&(d3.select(this).classed("hover",!1),u.elementMouseout({data:a,index:b,color:d3.select(this).style("fill")}))}).on("mousemove",function(a,b){v&&u.elementMousemove({data:a,index:b,color:d3.select(this).style("fill")})}).on("click",function(a,b){if(v){var c=this;u.eleme
ntClick({data:a,index:b,color:d3.select(this).style("fill"),event:d3.event,element:c}),d3.event.stopPropagation()}}).on("dblclick",function(a,b){v&&(u.elementDblClick({data:a,index:b,color:d3.select(this).style("fill")}),d3.event.stopPropagation())}),E.attr("fill",function(a,b){return t(a,b)}).attr("class",function(a,b,c){return(o(a,b)<0?"nv-bar negative":"nv-bar positive")+" nv-bar-"+c+"-"+b}).watchTransition(w,"bars").attr("transform",function(a,c){return"translate("+(l(n(a,c))-x/b[0].values.length*.45)+",0)"}).attr("width",x/b[0].values.length*.9),E.watchTransition(w,"bars").attr("y",function(b,c){var d=o(b,c)<0?m(0):m(0)-m(o(b,c))<1?m(0)-1:m(o(b,c));return a.utils.NaNtoZero(d)}).attr("height",function(b,c){return a.utils.NaNtoZero(Math.max(Math.abs(m(o(b,c))-m(0)),1))})}),w.renderEnd("historicalBar immediate"),b}var c,d,e,f,g={top:0,right:0,bottom:0,left:0},h=null,i=null,j=Math.floor(1e4*Math.random()),k=null,l=d3.scale.linear(),m=d3.scale.linear(),n=function(a){return a.x},o=fu
nction(a){return a.y},p=[],q=[0],r=!1,s=!0,t=a.utils.defaultColor(),u=d3.dispatch("chartClick","elementClick","elementDblClick","elementMouseover","elementMouseout","elementMousemove","renderEnd"),v=!0,w=a.utils.renderWatch(u,0);return b.highlightPoint=function(a,b){k.select(".nv-bars .nv-bar-0-"+a).classed("hover",b)},b.clearHighlights=function(){k.select(".nv-bars .nv-bar.hover").classed("hover",!1)},b.dispatch=u,b.options=a.utils.optionsFunc.bind(b),b._options=Object.create({},{width:{get:function(){return h},set:function(a){h=a}},height:{get:function(){return i},set:function(a){i=a}},forceX:{get:function(){return p},set:function(a){p=a}},forceY:{get:function(){return q},set:function(a){q=a}},padData:{get:function(){return r},set:function(a){r=a}},x:{get:function(){return n},set:function(a){n=a}},y:{get:function(){return o},set:function(a){o=a}},xScale:{get:function(){return l},set:function(a){l=a}},yScale:{get:function(){retur
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
deleted file mode 100644
index af0c1db..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/CBlockCli.java
+++ /dev/null
@@ -1,289 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.cli;
-
-import org.apache.commons.cli.BasicParser;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.OptionBuilder;
-import org.apache.commons.cli.Options;
-import org.apache.commons.cli.ParseException;
-import org.apache.hadoop.cblock.client.CBlockVolumeClient;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.util.Arrays;
-import java.util.List;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * The command line tool class.
- */
-public class CBlockCli extends Configured implements Tool {
-
- private static final String CREATE_VOLUME = "createVolume";
-
- private static final String DELETE_VOLUME = "deleteVolume";
-
- private static final String INFO_VOLUME = "infoVolume";
-
- private static final String LIST_VOLUME = "listVolume";
-
- private static final String SERVER_ADDR = "serverAddr";
-
- private static final String HELP = "help";
-
- private static final Logger LOG =
- LoggerFactory.getLogger(CBlockCli.class);
- private OzoneConfiguration conf;
-
- private PrintStream printStream;
-
- private Options options;
-
- private BasicParser parser;
-
- private CBlockVolumeClient localProxy;
-
- public CBlockCli(OzoneConfiguration conf, PrintStream printStream)
- throws IOException {
- this.printStream = printStream;
- this.conf = conf;
- this.options = getOptions();
- this.parser = new BasicParser();
- }
-
- public CBlockCli(OzoneConfiguration conf) throws IOException{
- this(conf, System.out);
- }
-
- private CommandLine parseArgs(String[] argv)
- throws ParseException {
- return parser.parse(options, argv);
- }
-
- private static Options getOptions() {
- Options options = new Options();
- Option serverAddress = OptionBuilder
- .withArgName("serverAddress>:<serverPort")
- .withLongOpt(SERVER_ADDR)
- .withValueSeparator(':')
- .hasArgs(2)
- .withDescription("specify server address:port")
- .create("s");
- options.addOption(serverAddress);
-
- // taking 4 args: userName, volumeName, volumeSize, blockSize
- Option createVolume = OptionBuilder
- .withArgName("user> <volume> <volumeSize in [GB/TB]> <blockSize")
- .withLongOpt(CREATE_VOLUME)
- .withValueSeparator(' ')
- .hasArgs(4)
- .withDescription("create a fresh new volume")
- .create("c");
- options.addOption(createVolume);
-
- // taking 2 args: userName, volumeName
- Option deleteVolume = OptionBuilder
- .withArgName("user> <volume")
- .withLongOpt(DELETE_VOLUME)
- .hasArgs(2)
- .withDescription("delete a volume")
- .create("d");
- options.addOption(deleteVolume);
-
- // taking 2 args: userName, volumeName
- Option infoVolume = OptionBuilder
- .withArgName("user> <volume")
- .withLongOpt(INFO_VOLUME)
- .hasArgs(2)
- .withDescription("info a volume")
- .create("i");
- options.addOption(infoVolume);
-
- // taking 1 arg: userName
- Option listVolume = OptionBuilder
- .withArgName("user")
- .withLongOpt(LIST_VOLUME)
- .hasOptionalArgs(1)
- .withDescription("list all volumes")
- .create("l");
- options.addOption(listVolume);
-
- Option help = OptionBuilder
- .withLongOpt(HELP)
- .withDescription("help")
- .create("h");
- options.addOption(help);
-
- return options;
- }
-
- @Override
- public int run(String[] args) throws ParseException, IOException {
- CommandLine commandLine = parseArgs(args);
- if (commandLine.hasOption("s")) {
- String[] serverAddrArgs = commandLine.getOptionValues("s");
- LOG.info("server address" + Arrays.toString(serverAddrArgs));
- String serverHost = serverAddrArgs[0];
- int serverPort = Integer.parseInt(serverAddrArgs[1]);
- InetSocketAddress serverAddress =
- new InetSocketAddress(serverHost, serverPort);
- this.localProxy = new CBlockVolumeClient(conf, serverAddress);
- } else {
- this.localProxy = new CBlockVolumeClient(conf);
- }
-
- if (commandLine.hasOption("h")) {
- LOG.info("help");
- help();
- }
-
- if (commandLine.hasOption("c")) {
- String[] createArgs = commandLine.getOptionValues("c");
- LOG.info("create volume:" + Arrays.toString(createArgs));
- createVolume(createArgs);
- }
-
- if (commandLine.hasOption("d")) {
- String[] deleteArgs = commandLine.getOptionValues("d");
- LOG.info("delete args:" + Arrays.toString(deleteArgs));
- deleteVolume(deleteArgs);
- }
-
- if (commandLine.hasOption("l")) {
- String[] listArg = commandLine.getOptionValues("l");
- LOG.info("list args:" + Arrays.toString(listArg));
- listVolume(listArg);
- }
-
- if (commandLine.hasOption("i")) {
- String[] infoArgs = commandLine.getOptionValues("i");
- LOG.info("info args:" + Arrays.toString(infoArgs));
- infoVolume(infoArgs);
- }
- return 0;
- }
-
- public static void main(String[] argv) throws Exception {
- OzoneConfiguration cblockConf = new OzoneConfiguration();
- RPC.setProtocolEngine(cblockConf, CBlockServiceProtocolPB.class,
- ProtobufRpcEngine.class);
- int res = 0;
- Tool shell = new CBlockCli(cblockConf, System.out);
- try {
- ToolRunner.run(shell, argv);
- } catch (Exception ex) {
- LOG.error(ex.toString());
- res = 1;
- }
- System.exit(res);
- }
-
- public static long parseSize(String volumeSizeArgs) throws IOException {
- long multiplier = 1;
-
- Pattern p = Pattern.compile("([0-9]+)([a-zA-Z]+)");
- Matcher m = p.matcher(volumeSizeArgs);
-
- if (!m.find()) {
- throw new IOException("Invalid volume size args " + volumeSizeArgs);
- }
-
- int size = Integer.parseInt(m.group(1));
- String s = m.group(2);
-
- if (s.equalsIgnoreCase("MB") ||
- s.equalsIgnoreCase("Mi")) {
- multiplier = 1024L * 1024;
- } else if (s.equalsIgnoreCase("GB") ||
- s.equalsIgnoreCase("Gi")) {
- multiplier = 1024L * 1024 * 1024;
- } else if (s.equalsIgnoreCase("TB") ||
- s.equalsIgnoreCase("Ti")) {
- multiplier = 1024L * 1024 * 1024 * 1024;
- } else {
- throw new IOException("Invalid volume size args " + volumeSizeArgs);
- }
- return size * multiplier;
- }
-
- private void createVolume(String[] createArgs) throws IOException {
- String userName = createArgs[0];
- String volumeName = createArgs[1];
- long volumeSize = parseSize(createArgs[2]);
- int blockSize = Integer.parseInt(createArgs[3])*1024;
- localProxy.createVolume(userName, volumeName, volumeSize, blockSize);
- }
-
- private void deleteVolume(String[] deleteArgs) throws IOException {
- String userName = deleteArgs[0];
- String volumeName = deleteArgs[1];
- boolean force = false;
- if (deleteArgs.length > 2) {
- force = Boolean.parseBoolean(deleteArgs[2]);
- }
- localProxy.deleteVolume(userName, volumeName, force);
- }
-
- private void infoVolume(String[] infoArgs) throws IOException {
- String userName = infoArgs[0];
- String volumeName = infoArgs[1];
- VolumeInfo volumeInfo = localProxy.infoVolume(userName, volumeName);
- printStream.println(volumeInfo.toString());
- }
-
- private void listVolume(String[] listArgs) throws IOException {
- StringBuilder stringBuilder = new StringBuilder();
- List<VolumeInfo> volumeResponse;
- if (listArgs == null) {
- volumeResponse = localProxy.listVolume(null);
- } else {
- volumeResponse = localProxy.listVolume(listArgs[0]);
- }
- for (int i = 0; i<volumeResponse.size(); i++) {
- stringBuilder.append(
- String.format("%s:%s\t%d\t%d", volumeResponse.get(i).getUserName(),
- volumeResponse.get(i).getVolumeName(),
- volumeResponse.get(i).getVolumeSize(),
- volumeResponse.get(i).getBlockSize()));
- if (i < volumeResponse.size() - 1) {
- stringBuilder.append("\n");
- }
- }
- printStream.println(stringBuilder);
- }
-
- private void help() {
- HelpFormatter formatter = new HelpFormatter();
- formatter.printHelp(100, "cblock", "", options, "");
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
deleted file mode 100644
index b8b1889..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/cli/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.cli;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
deleted file mode 100644
index ec38c9c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockServiceProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.client;
-
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockServiceProtocol;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * The client side implement of CBlockServiceProtocol.
- */
-@InterfaceAudience.Private
-public final class CBlockServiceProtocolClientSideTranslatorPB
- implements CBlockServiceProtocol, ProtocolTranslator, Closeable {
-
- private final CBlockServiceProtocolPB rpcProxy;
-
- public CBlockServiceProtocolClientSideTranslatorPB(
- CBlockServiceProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- @Override
- public void close() throws IOException {
- RPC.stopProxy(rpcProxy);
- }
-
- @Override
- public void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws IOException {
- CBlockServiceProtocolProtos.CreateVolumeRequestProto.Builder req =
- CBlockServiceProtocolProtos.CreateVolumeRequestProto.newBuilder();
- req.setUserName(userName);
- req.setVolumeName(volumeName);
- req.setVolumeSize(volumeSize);
- req.setBlockSize(blockSize);
- try {
- rpcProxy.createVolume(null, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
- public void deleteVolume(String userName, String volumeName, boolean force)
- throws IOException {
- CBlockServiceProtocolProtos.DeleteVolumeRequestProto.Builder req =
- CBlockServiceProtocolProtos.DeleteVolumeRequestProto.newBuilder();
- req.setUserName(userName);
- req.setVolumeName(volumeName);
- req.setForce(force);
- try {
- rpcProxy.deleteVolume(null, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public VolumeInfo infoVolume(String userName, String volumeName)
- throws IOException {
- CBlockServiceProtocolProtos.InfoVolumeRequestProto.Builder req =
- CBlockServiceProtocolProtos.InfoVolumeRequestProto.newBuilder();
- req.setUserName(userName);
- req.setVolumeName(volumeName);
- try {
- CBlockServiceProtocolProtos.InfoVolumeResponseProto resp =
- rpcProxy.infoVolume(null, req.build());
- return new VolumeInfo(resp.getVolumeInfo().getUserName(),
- resp.getVolumeInfo().getVolumeName(),
- resp.getVolumeInfo().getVolumeSize(),
- resp.getVolumeInfo().getBlockSize(),
- resp.getVolumeInfo().getUsage());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
- public List<VolumeInfo> listVolume(String userName) throws IOException {
- CBlockServiceProtocolProtos.ListVolumeRequestProto.Builder req =
- CBlockServiceProtocolProtos.ListVolumeRequestProto.newBuilder();
- if (userName != null) {
- req.setUserName(userName);
- }
- try {
- CBlockServiceProtocolProtos.ListVolumeResponseProto resp =
- rpcProxy.listVolume(null, req.build());
- List<VolumeInfo> respList = new ArrayList<>();
- for (CBlockServiceProtocolProtos.VolumeInfoProto entry :
- resp.getVolumeEntryList()) {
- VolumeInfo volumeInfo = new VolumeInfo(
- entry.getUserName(), entry.getVolumeName(), entry.getVolumeSize(),
- entry.getBlockSize());
- respList.add(volumeInfo);
- }
- return respList;
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- } catch (Exception e) {
- throw new IOException("got" + e.getCause() + " " + e.getMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
deleted file mode 100644
index 9ed54e3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/CBlockVolumeClient.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.client;
-
-import org.apache.hadoop.cblock.CBlockConfigKeys;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.protocolPB.CBlockServiceProtocolPB;
-import org.apache.hadoop.io.retry.RetryPolicies;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.ozone.client.OzoneClientUtils;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.security.UserGroupInformation;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Implementation of client used by CBlock command line tool.
- */
-public class CBlockVolumeClient {
- private final CBlockServiceProtocolClientSideTranslatorPB cblockClient;
-
- public CBlockVolumeClient(OzoneConfiguration conf) throws IOException {
- this(conf, null);
- }
-
- public CBlockVolumeClient(OzoneConfiguration conf,
- InetSocketAddress serverAddress) throws IOException {
- InetSocketAddress address = serverAddress != null ? serverAddress :
- OzoneClientUtils.getCblockServiceRpcAddr(conf);
- long version = RPC.getProtocolVersion(CBlockServiceProtocolPB.class);
- int rpcTimeout = Math.toIntExact(
- conf.getTimeDuration(CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT,
- CBlockConfigKeys.DFS_CBLOCK_RPC_TIMEOUT_DEFAULT,
- TimeUnit.MILLISECONDS));
- cblockClient = new CBlockServiceProtocolClientSideTranslatorPB(
- RPC.getProtocolProxy(CBlockServiceProtocolPB.class, version,
- address, UserGroupInformation.getCurrentUser(), conf,
- NetUtils.getDefaultSocketFactory(conf), rpcTimeout, RetryPolicies
- .retryUpToMaximumCountWithFixedSleep(
- 300, 1, TimeUnit.SECONDS)).getProxy());
- }
-
- public void createVolume(String userName, String volumeName,
- long volumeSize, int blockSize) throws IOException {
- cblockClient.createVolume(userName, volumeName,
- volumeSize, blockSize);
- }
-
- public void deleteVolume(String userName, String volumeName, boolean force)
- throws IOException {
- cblockClient.deleteVolume(userName, volumeName, force);
- }
-
- public VolumeInfo infoVolume(String userName, String volumeName)
- throws IOException {
- return cblockClient.infoVolume(userName, volumeName);
- }
-
- public List<VolumeInfo> listVolume(String userName)
- throws IOException {
- return cblockClient.listVolume(userName);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
deleted file mode 100644
index 761b71e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/client/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.client;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
deleted file mode 100644
index 8f6b82b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/CBlockException.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.exception;
-
-import java.io.IOException;
-
-/**
- * The exception class used in CBlock.
- */
-public class CBlockException extends IOException {
- public CBlockException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
deleted file mode 100644
index 268b8cb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/exception/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.exception;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
deleted file mode 100644
index 04fe3a4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/BlockWriterTask.java
+++ /dev/null
@@ -1,175 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.base.Preconditions;
-import com.google.common.primitives.Longs;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.AsyncBlockWriter;
-import org.apache.hadoop.scm.XceiverClientSpi;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.storage.ContainerProtocolCalls;
-import org.apache.hadoop.util.Time;
-import org.apache.hadoop.utils.LevelDBStore;
-
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Paths;
-
-/**
- * The blockWriter task.
- */
-public class BlockWriterTask implements Runnable {
- private final LogicalBlock block;
- private int tryCount;
- private final ContainerCacheFlusher flusher;
- private final String dbPath;
- private final String fileName;
- private final int maxRetryCount;
-
- /**
- * Constructs a BlockWriterTask.
- *
- * @param block - Block Information.
- * @param flusher - ContainerCacheFlusher.
- */
- public BlockWriterTask(LogicalBlock block, ContainerCacheFlusher flusher,
- String dbPath, int tryCount, String fileName, int maxRetryCount) {
- this.block = block;
- this.flusher = flusher;
- this.dbPath = dbPath;
- this.tryCount = tryCount;
- this.fileName = fileName;
- this.maxRetryCount = maxRetryCount;
- }
-
- /**
- * When an object implementing interface <code>Runnable</code> is used
- * to create a thread, starting the thread causes the object's
- * <code>run</code> method to be called in that separately executing
- * thread.
- * <p>
- * The general contract of the method <code>run</code> is that it may
- * take any action whatsoever.
- *
- * @see Thread#run()
- */
- @Override
- public void run() {
- String containerName = null;
- XceiverClientSpi client = null;
- LevelDBStore levelDBStore = null;
- String traceID = flusher.getTraceID(new File(dbPath), block.getBlockID());
- flusher.getLOG().debug(
- "Writing block to remote. block ID: {}", block.getBlockID());
- try {
- incTryCount();
- Pipeline pipeline = flusher.getPipeline(this.dbPath, block.getBlockID());
- client = flusher.getXceiverClientManager().acquireClient(pipeline);
- containerName = pipeline.getContainerName();
- byte[] keybuf = Longs.toByteArray(block.getBlockID());
- byte[] data;
- long startTime = Time.monotonicNow();
- levelDBStore = flusher.getCacheDB(this.dbPath);
- data = levelDBStore.get(keybuf);
- Preconditions.checkNotNull(data);
- long endTime = Time.monotonicNow();
- Preconditions.checkState(data.length > 0, "Block data is zero length");
- startTime = Time.monotonicNow();
- ContainerProtocolCalls.writeSmallFile(client, containerName,
- Long.toString(block.getBlockID()), data, traceID);
- endTime = Time.monotonicNow();
- flusher.getTargetMetrics().updateContainerWriteLatency(
- endTime - startTime);
- flusher.getLOG().debug("Time taken for Write Small File : {} ms",
- endTime - startTime);
-
- flusher.incrementRemoteIO();
-
- } catch (Exception ex) {
- flusher.getLOG().error("Writing of block:{} failed, We have attempted " +
- "to write this block {} times to the container {}.Trace ID:{}",
- block.getBlockID(), this.getTryCount(), containerName, traceID, ex);
- writeRetryBlock(block);
- if (ex instanceof IOException) {
- flusher.getTargetMetrics().incNumWriteIOExceptionRetryBlocks();
- } else {
- flusher.getTargetMetrics().incNumWriteGenericExceptionRetryBlocks();
- }
- if (this.getTryCount() >= maxRetryCount) {
- flusher.getTargetMetrics().incNumWriteMaxRetryBlocks();
- }
- } finally {
- flusher.incFinishCount(fileName);
- if (levelDBStore != null) {
- flusher.releaseCacheDB(dbPath);
- }
- if(client != null) {
- flusher.getXceiverClientManager().releaseClient(client);
- }
- }
- }
-
-
- private void writeRetryBlock(LogicalBlock currentBlock) {
- boolean append = false;
- String retryFileName =
- String.format("%s.%d.%s.%s", AsyncBlockWriter.RETRY_LOG_PREFIX,
- currentBlock.getBlockID(), Time.monotonicNow(), tryCount);
- File logDir = new File(this.dbPath);
- if (!logDir.exists() && !logDir.mkdirs()) {
- flusher.getLOG().error(
- "Unable to create the log directory, Critical error cannot continue");
- return;
- }
- String log = Paths.get(this.dbPath, retryFileName).toString();
- ByteBuffer buffer = ByteBuffer.allocate(Long.SIZE / Byte.SIZE);
- buffer.putLong(currentBlock.getBlockID());
- buffer.flip();
- try {
- FileChannel channel = new FileOutputStream(log, append).getChannel();
- channel.write(buffer);
- channel.close();
- flusher.processDirtyBlocks(this.dbPath, retryFileName);
- } catch (IOException e) {
- flusher.getTargetMetrics().incNumFailedRetryLogFileWrites();
- flusher.getLOG().error("Unable to write the retry block. Block ID: {}",
- currentBlock.getBlockID(), e);
- }
- }
-
- /**
- * Increments the try count. This is done each time we try this block
- * write to the container.
- */
- private void incTryCount() {
- tryCount++;
- }
-
- /**
- * Get the retry count.
- *
- * @return int
- */
- public int getTryCount() {
- return tryCount;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 84b68e3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockClientProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.primitives.Longs;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.cblock.exception.CBlockException;
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.CBlockClientProtocol;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos.ContainerIDProto;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos.ListVolumesRequestProto;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos.ListVolumesResponseProto;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos.MountVolumeRequestProto;
-import org.apache.hadoop.cblock.protocol.proto
- .CBlockClientServerProtocolProtos.MountVolumeResponseProto;
-import org.apache.hadoop.cblock.protocol.proto.CBlockServiceProtocolProtos
- .VolumeInfoProto;
-import org.apache.hadoop.cblock.protocolPB.CBlockClientServerProtocolPB;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-
-/**
- * The client side of CBlockClientProtocol.
- *
- * CBlockClientProtocol is the protocol used between cblock client side
- * and cblock manager (cblock client side is just the node where jscsi daemon
- * process runs. a machines talks to jscsi daemon for mounting a volume).
- *
- * Right now, the only communication carried by this protocol is for client side
- * to request mounting a volume.
- */
-public class CBlockClientProtocolClientSideTranslatorPB
- implements CBlockClientProtocol, ProtocolTranslator, Closeable {
-
- private final CBlockClientServerProtocolPB rpcProxy;
-
- public CBlockClientProtocolClientSideTranslatorPB(
- CBlockClientServerProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
-
- @Override
- public void close() throws IOException {
- RPC.stopProxy(rpcProxy);
- }
-
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- @Override
- public MountVolumeResponse mountVolume(
- String userName, String volumeName) throws IOException {
- MountVolumeRequestProto.Builder
- request
- = MountVolumeRequestProto
- .newBuilder();
- request.setUserName(userName);
- request.setVolumeName(volumeName);
- try {
- MountVolumeResponseProto resp
- = rpcProxy.mountVolume(null, request.build());
- if (!resp.getIsValid()) {
- throw new CBlockException(
- "Not a valid volume:" + userName + ":" + volumeName);
- }
- List<Pipeline> containerIDs = new ArrayList<>();
- HashMap<String, Pipeline> containerPipelines = new HashMap<>();
- if (resp.getAllContainerIDsList().size() == 0) {
- throw new CBlockException("Mount volume request returned no container");
- }
- for (ContainerIDProto containerID :
- resp.getAllContainerIDsList()) {
- if (containerID.hasPipeline()) {
- // it should always have a pipeline only except for tests.
- Pipeline p = Pipeline.getFromProtoBuf(containerID.getPipeline());
- p.setData(Longs.toByteArray(containerID.getIndex()));
- containerIDs.add(p);
- containerPipelines.put(containerID.getContainerID(), p);
- } else {
- throw new CBlockException("ContainerID does not have pipeline!");
- }
- }
- return new MountVolumeResponse(
- resp.getIsValid(),
- resp.getUserName(),
- resp.getVolumeName(),
- resp.getVolumeSize(),
- resp.getBlockSize(),
- containerIDs,
- containerPipelines);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-
- @Override
- public List<VolumeInfo> listVolumes() throws IOException {
- try {
- List<VolumeInfo> result = new ArrayList<>();
- ListVolumesResponseProto
- listVolumesResponseProto = this.rpcProxy.listVolumes(null,
- ListVolumesRequestProto.newBuilder()
- .build());
- for (VolumeInfoProto volumeInfoProto :
- listVolumesResponseProto
- .getVolumeEntryList()) {
- result.add(new VolumeInfo(volumeInfoProto.getUserName(),
- volumeInfoProto.getVolumeName(), volumeInfoProto.getVolumeSize(),
- volumeInfoProto.getBlockSize()));
- }
- return result;
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
deleted file mode 100644
index 2f35668..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockIStorageImpl.java
+++ /dev/null
@@ -1,440 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.cblock.jscsiHelper.cache.CacheModule;
-import org.apache.hadoop.cblock.jscsiHelper.cache.LogicalBlock;
-import org.apache.hadoop.cblock.jscsiHelper.cache.impl.CBlockLocalCache;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.jscsi.target.storage.IStorageModule;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.apache.hadoop.cblock.CBlockConfigKeys.DFS_CBLOCK_TRACE_IO;
-import static org.apache.hadoop.cblock.CBlockConfigKeys
- .DFS_CBLOCK_TRACE_IO_DEFAULT;
-
-/**
- * The SCSI Target class for CBlockSCSIServer.
- */
-final public class CBlockIStorageImpl implements IStorageModule {
- private static final Logger LOGGER =
- LoggerFactory.getLogger(CBlockIStorageImpl.class);
- private static final Logger TRACER =
- LoggerFactory.getLogger("TraceIO");
-
- private CacheModule cache;
- private final long volumeSize;
- private final int blockSize;
- private final String userName;
- private final String volumeName;
- private final boolean traceEnabled;
- private final Configuration conf;
- private final ContainerCacheFlusher flusher;
- private List<Pipeline> fullContainerList;
-
- /**
- * private: constructs a SCSI Target.
- *
- * @param config - config
- * @param userName - Username
- * @param volumeName - Name of the volume
- * @param volumeSize - Size of the volume
- * @param blockSize - Size of the block
- * @param fullContainerList - Ordered list of containers that make up this
- * volume.
- * @param flusher - flusher which is used to flush data from
- * level db cache to containers
- * @throws IOException - Throws IOException.
- */
- private CBlockIStorageImpl(Configuration config, String userName,
- String volumeName, long volumeSize, int blockSize,
- List<Pipeline> fullContainerList, ContainerCacheFlusher flusher) {
- this.conf = config;
- this.userName = userName;
- this.volumeName = volumeName;
- this.volumeSize = volumeSize;
- this.blockSize = blockSize;
- this.fullContainerList = new ArrayList<>(fullContainerList);
- this.flusher = flusher;
- this.traceEnabled = conf.getBoolean(DFS_CBLOCK_TRACE_IO,
- DFS_CBLOCK_TRACE_IO_DEFAULT);
- }
-
- /**
- * private: initialize the cache.
- *
- * @param xceiverClientManager - client manager that is used for creating new
- * connections to containers.
- * @param metrics - target metrics to maintain metrics for target server
- * @throws IOException - Throws IOException.
- */
- private void initCache(XceiverClientManager xceiverClientManager,
- CBlockTargetMetrics metrics) throws IOException {
- this.cache = CBlockLocalCache.newBuilder()
- .setConfiguration(conf)
- .setVolumeName(this.volumeName)
- .setUserName(this.userName)
- .setPipelines(this.fullContainerList)
- .setClientManager(xceiverClientManager)
- .setBlockSize(blockSize)
- .setVolumeSize(volumeSize)
- .setFlusher(flusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- this.cache.start();
- }
-
- /**
- * Gets a new builder for CBlockStorageImpl.
- *
- * @return builder
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Get Cache.
- *
- * @return - Cache
- */
- public CacheModule getCache() {
- return cache;
- }
-
- /**
- * Returns block size of this volume.
- *
- * @return int size of block for this volume.
- */
- @Override
- public int getBlockSize() {
- return blockSize;
- }
-
- /**
- * Checks the index boundary of a block address.
- *
- * @param logicalBlockAddress the index of the first block of data to be read
- * or written
- * @param transferLengthInBlocks the total number of consecutive blocks about
- * to be read or written
- * @return 0 == Success, 1 indicates the LBA address is out of bounds and 2
- * indicates that LBA + transfer size is out of bounds.
- */
- @Override
- public int checkBounds(long logicalBlockAddress, int transferLengthInBlocks) {
- long sizeInBlocks = volumeSize / blockSize;
- int res = 0;
- if (logicalBlockAddress < 0 || logicalBlockAddress >= sizeInBlocks) {
- res = 1;
- }
-
- if (transferLengthInBlocks < 0 ||
- logicalBlockAddress + transferLengthInBlocks > sizeInBlocks) {
- if (res == 0) {
- res = 2;
- }
- }
- return res;
- }
-
- /**
- * Number of blocks that make up this volume.
- *
- * @return long - count of blocks.
- */
- @Override
- public long getSizeInBlocks() {
- return volumeSize / blockSize;
- }
-
- /**
- * Reads the number of bytes that can be read into the bytes buffer from the
- * location indicated.
- *
- * @param bytes the array into which the data will be copied will be filled
- * with data from storage
- * @param storageIndex the position of the first byte to be copied
- * @throws IOException
- */
- @Override
- public void read(byte[] bytes, long storageIndex) throws IOException {
- int startingIdxInBlock = (int) storageIndex % blockSize;
- int idxInBytes = 0;
- if (this.traceEnabled) {
- TRACER.info("Task=ReadStart,length={},location={}",
- bytes.length, storageIndex);
- }
- while (idxInBytes < bytes.length - 1) {
- long blockId = (storageIndex + idxInBytes) / blockSize;
- byte[] dataBytes;
-
- try {
- LogicalBlock block = this.cache.get(blockId);
- dataBytes = block.getData().array();
-
- if (this.traceEnabled) {
- TRACER.info("Task=ReadBlock,BlockID={},length={},SHA={}",
- blockId,
- dataBytes.length,
- dataBytes.length > 0 ? DigestUtils.sha256Hex(dataBytes) : null);
- }
- } catch (IOException e) {
- // For an non-existing block cache.get will return a block with zero
- // bytes filled. So any error here is a real error.
- LOGGER.error("getting errors when reading data:" + e);
- throw e;
- }
-
- int length = blockSize - startingIdxInBlock;
- if (length > bytes.length - idxInBytes) {
- length = bytes.length - idxInBytes;
- }
- if (dataBytes.length >= length) {
- System.arraycopy(dataBytes, startingIdxInBlock, bytes, idxInBytes,
- length);
- }
- startingIdxInBlock = 0;
- idxInBytes += length;
- }
- if (this.traceEnabled) {
- TRACER.info("Task=ReadEnd,length={},location={},SHA={}",
- bytes.length, storageIndex, DigestUtils.sha256Hex(bytes));
- }
- }
-
- @Override
- public void write(byte[] bytes, long storageIndex) throws IOException {
- int startingIdxInBlock = (int) storageIndex % blockSize;
- int idxInBytes = 0;
- if (this.traceEnabled) {
- TRACER.info("Task=WriteStart,length={},location={},SHA={}",
- bytes.length, storageIndex,
- bytes.length > 0 ? DigestUtils.sha256Hex(bytes) : null);
- }
-
- ByteBuffer dataByte = ByteBuffer.allocate(blockSize);
- while (idxInBytes < bytes.length - 1) {
- long blockId = (storageIndex + idxInBytes) / blockSize;
- int length = blockSize - startingIdxInBlock;
- if (length > bytes.length - idxInBytes) {
- length = bytes.length - idxInBytes;
- }
- System.arraycopy(bytes, idxInBytes, dataByte.array(), startingIdxInBlock,
- length);
- this.cache.put(blockId, dataByte.array());
-
- if (this.traceEnabled) {
- TRACER.info("Task=WriteBlock,BlockID={},length={},SHA={}",
- blockId, dataByte.array().length,
- dataByte.array().length > 0 ?
- DigestUtils.sha256Hex(dataByte.array()) : null);
- }
- dataByte.clear();
- startingIdxInBlock = 0;
- idxInBytes += length;
- }
-
- if (this.traceEnabled) {
- TRACER.info("Task=WriteEnd,length={},location={} ",
- bytes.length, storageIndex);
- }
- }
-
- @Override
- public void close() throws IOException {
- try {
- cache.close();
- } catch (IllegalStateException ise) {
- LOGGER.error("Can not close the storage {}", ise);
- throw ise;
- }
- }
-
- /**
- * Builder class for CBlocklocalCache.
- */
- public static class Builder {
- private String userName;
- private String volumeName;
- private long volumeSize;
- private int blockSize;
- private List<Pipeline> containerList;
- private Configuration conf;
- private XceiverClientManager clientManager;
- private ContainerCacheFlusher flusher;
- private CBlockTargetMetrics metrics;
-
- /**
- * Constructs a builder.
- */
- Builder() {
-
- }
-
- public Builder setFlusher(ContainerCacheFlusher cacheFlusher) {
- this.flusher = cacheFlusher;
- return this;
- }
-
- /**
- * set config.
- *
- * @param config - config
- * @return Builder
- */
- public Builder setConf(Configuration config) {
- this.conf = config;
- return this;
- }
-
- /**
- * set user name.
- *
- * @param cblockUserName - user name
- * @return Builder
- */
- public Builder setUserName(String cblockUserName) {
- this.userName = cblockUserName;
- return this;
- }
-
- /**
- * set volume name.
- *
- * @param cblockVolumeName -- volume name
- * @return Builder
- */
- public Builder setVolumeName(String cblockVolumeName) {
- this.volumeName = cblockVolumeName;
- return this;
- }
-
- /**
- * set volume size.
- *
- * @param cblockVolumeSize -- set volume size.
- * @return Builder
- */
- public Builder setVolumeSize(long cblockVolumeSize) {
- this.volumeSize = cblockVolumeSize;
- return this;
- }
-
- /**
- * set block size.
- *
- * @param cblockBlockSize -- block size
- * @return Builder
- */
- public Builder setBlockSize(int cblockBlockSize) {
- this.blockSize = cblockBlockSize;
- return this;
- }
-
- /**
- * Set contianer list.
- *
- * @param cblockContainerList - set the pipeline list
- * @return Builder
- */
- public Builder setContainerList(List<Pipeline> cblockContainerList) {
- this.containerList = cblockContainerList;
- return this;
- }
-
- /**
- * Set client manager.
- *
- * @param xceiverClientManager -- sets the client manager.
- * @return Builder
- */
- public Builder setClientManager(XceiverClientManager xceiverClientManager) {
- this.clientManager = xceiverClientManager;
- return this;
- }
-
- /**
- * Set Cblock Target Metrics.
- *
- * @param targetMetrics -- sets the cblock target metrics
- * @return Builder
- */
- public Builder setCBlockTargetMetrics(CBlockTargetMetrics targetMetrics) {
- this.metrics = targetMetrics;
- return this;
- }
-
- /**
- * Builds the CBlockStorageImpl.
- *
- * @return builds the CBlock Scsi Target.
- */
- public CBlockIStorageImpl build() throws IOException {
- if (StringUtils.isBlank(userName)) {
- throw new IllegalArgumentException("User name cannot be null or empty" +
- ".");
- }
- if (StringUtils.isBlank(volumeName)) {
- throw new IllegalArgumentException("Volume name cannot be null or " +
- "empty");
- }
-
- if (volumeSize < 1) {
- throw new IllegalArgumentException("Volume size cannot be negative or" +
- " zero.");
- }
-
- if (blockSize < 1) {
- throw new IllegalArgumentException("Block size cannot be negative or " +
- "zero.");
- }
-
- if (containerList == null || containerList.size() == 0) {
- throw new IllegalArgumentException("Container list cannot be null or " +
- "empty");
- }
- if (clientManager == null) {
- throw new IllegalArgumentException("Client manager cannot be null");
- }
- if (conf == null) {
- throw new IllegalArgumentException("Configuration cannot be null");
- }
-
- if (flusher == null) {
- throw new IllegalArgumentException("Flusher Cannot be null.");
- }
- CBlockIStorageImpl impl = new CBlockIStorageImpl(this.conf, this.userName,
- this.volumeName, this.volumeSize, this.blockSize, this.containerList,
- this.flusher);
- impl.initCache(this.clientManager, this.metrics);
- return impl;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
deleted file mode 100644
index 6367c61..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockManagerHandler.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import org.apache.hadoop.cblock.meta.VolumeInfo;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * This class is the handler of CBlockManager used by target server
- * to communicate with CBlockManager.
- *
- * More specifically, this class will expose local methods to target
- * server, and make RPC calls to CBlockManager accordingly
- */
-public class CBlockManagerHandler {
-
- private final CBlockClientProtocolClientSideTranslatorPB handler;
-
- public CBlockManagerHandler(
- CBlockClientProtocolClientSideTranslatorPB handler) {
- this.handler = handler;
- }
-
- public MountVolumeResponse mountVolume(
- String userName, String volumeName) throws IOException {
- return handler.mountVolume(userName, volumeName);
- }
-
- public List<VolumeInfo> listVolumes() throws IOException {
- return handler.listVolumes();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
deleted file mode 100644
index e7df0cf..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetMetrics.java
+++ /dev/null
@@ -1,334 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.metrics2.MetricsSystem;
-import org.apache.hadoop.metrics2.annotation.Metric;
-import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
-import org.apache.hadoop.metrics2.lib.MutableCounterLong;
-import org.apache.hadoop.metrics2.lib.MutableRate;
-
-/**
- * This class is for maintaining the various Cblock Target statistics
- * and publishing them through the metrics interfaces.
- * This also registers the JMX MBean for RPC.
- *
- * This class maintains stats like cache hit and miss ratio
- * as well as the latency time of read and write ops.
- */
-public class CBlockTargetMetrics {
- // IOPS based Metrics
- @Metric private MutableCounterLong numReadOps;
- @Metric private MutableCounterLong numWriteOps;
- @Metric private MutableCounterLong numReadCacheHits;
- @Metric private MutableCounterLong numReadCacheMiss;
- @Metric private MutableCounterLong numDirectBlockWrites;
-
- // Cblock internal Metrics
- @Metric private MutableCounterLong numDirtyLogBlockRead;
- @Metric private MutableCounterLong numBytesDirtyLogRead;
- @Metric private MutableCounterLong numBytesDirtyLogWritten;
- @Metric private MutableCounterLong numBlockBufferFlushCompleted;
- @Metric private MutableCounterLong numBlockBufferFlushTriggered;
- @Metric private MutableCounterLong numBlockBufferUpdates;
- @Metric private MutableCounterLong numRetryLogBlockRead;
- @Metric private MutableCounterLong numBytesRetryLogRead;
-
- // Failure Metrics
- @Metric private MutableCounterLong numReadLostBlocks;
- @Metric private MutableCounterLong numFailedReadBlocks;
- @Metric private MutableCounterLong numWriteIOExceptionRetryBlocks;
- @Metric private MutableCounterLong numWriteGenericExceptionRetryBlocks;
- @Metric private MutableCounterLong numFailedDirectBlockWrites;
- @Metric private MutableCounterLong numIllegalDirtyLogFiles;
- @Metric private MutableCounterLong numFailedDirtyLogFileDeletes;
- @Metric private MutableCounterLong numFailedBlockBufferFlushes;
- @Metric private MutableCounterLong numInterruptedBufferWaits;
- @Metric private MutableCounterLong numFailedRetryLogFileWrites;
- @Metric private MutableCounterLong numWriteMaxRetryBlocks;
- @Metric private MutableCounterLong numFailedReleaseLevelDB;
-
- // Latency based Metrics
- @Metric private MutableRate dbReadLatency;
- @Metric private MutableRate containerReadLatency;
- @Metric private MutableRate dbWriteLatency;
- @Metric private MutableRate containerWriteLatency;
- @Metric private MutableRate blockBufferFlushLatency;
- @Metric private MutableRate directBlockWriteLatency;
-
- public CBlockTargetMetrics() {
- }
-
- public static CBlockTargetMetrics create() {
- MetricsSystem ms = DefaultMetricsSystem.instance();
- return ms.register("CBlockTargetMetrics",
- "CBlock Target Metrics",
- new CBlockTargetMetrics());
- }
-
- public void incNumReadOps() {
- numReadOps.incr();
- }
-
- public void incNumWriteOps() {
- numWriteOps.incr();
- }
-
- public void incNumReadCacheHits() {
- numReadCacheHits.incr();
- }
-
- public void incNumReadCacheMiss() {
- numReadCacheMiss.incr();
- }
-
- public void incNumReadLostBlocks() {
- numReadLostBlocks.incr();
- }
-
- public void incNumDirectBlockWrites() {
- numDirectBlockWrites.incr();
- }
-
- public void incNumWriteIOExceptionRetryBlocks() {
- numWriteIOExceptionRetryBlocks.incr();
- }
-
- public void incNumWriteGenericExceptionRetryBlocks() {
- numWriteGenericExceptionRetryBlocks.incr();
- }
-
- public void incNumFailedDirectBlockWrites() {
- numFailedDirectBlockWrites.incr();
- }
-
- public void incNumFailedReadBlocks() {
- numFailedReadBlocks.incr();
- }
-
- public void incNumBlockBufferFlushCompleted() {
- numBlockBufferFlushCompleted.incr();
- }
-
- public void incNumBlockBufferFlushTriggered() {
- numBlockBufferFlushTriggered.incr();
- }
-
- public void incNumDirtyLogBlockRead() {
- numDirtyLogBlockRead.incr();
- }
-
- public void incNumBytesDirtyLogRead(int bytes) {
- numBytesDirtyLogRead.incr(bytes);
- }
-
- public void incNumBlockBufferUpdates() {
- numBlockBufferUpdates.incr();
- }
-
- public void incNumRetryLogBlockRead() {
- numRetryLogBlockRead.incr();
- }
-
- public void incNumBytesRetryLogRead(int bytes) {
- numBytesRetryLogRead.incr(bytes);
- }
-
- public void incNumBytesDirtyLogWritten(int bytes) {
- numBytesDirtyLogWritten.incr(bytes);
- }
-
- public void incNumFailedBlockBufferFlushes() {
- numFailedBlockBufferFlushes.incr();
- }
-
- public void incNumInterruptedBufferWaits() {
- numInterruptedBufferWaits.incr();
- }
-
- public void incNumIllegalDirtyLogFiles() {
- numIllegalDirtyLogFiles.incr();
- }
-
- public void incNumFailedDirtyLogFileDeletes() {
- numFailedDirtyLogFileDeletes.incr();
- }
-
- public void incNumFailedRetryLogFileWrites() {
- numFailedRetryLogFileWrites.incr();
- }
-
- public void incNumWriteMaxRetryBlocks() {
- numWriteMaxRetryBlocks.incr();
- }
-
- public void incNumFailedReleaseLevelDB() {
- numFailedReleaseLevelDB.incr();
- }
-
- public void updateDBReadLatency(long latency) {
- dbReadLatency.add(latency);
- }
-
- public void updateContainerReadLatency(long latency) {
- containerReadLatency.add(latency);
- }
-
- public void updateDBWriteLatency(long latency) {
- dbWriteLatency.add(latency);
- }
-
- public void updateContainerWriteLatency(long latency) {
- containerWriteLatency.add(latency);
- }
-
- public void updateDirectBlockWriteLatency(long latency) {
- directBlockWriteLatency.add(latency);
- }
-
- public void updateBlockBufferFlushLatency(long latency) {
- blockBufferFlushLatency.add(latency);
- }
-
- @VisibleForTesting
- public long getNumReadOps() {
- return numReadOps.value();
- }
-
- @VisibleForTesting
- public long getNumWriteOps() {
- return numWriteOps.value();
- }
-
- @VisibleForTesting
- public long getNumReadCacheHits() {
- return numReadCacheHits.value();
- }
-
- @VisibleForTesting
- public long getNumReadCacheMiss() {
- return numReadCacheMiss.value();
- }
-
- @VisibleForTesting
- public long getNumReadLostBlocks() {
- return numReadLostBlocks.value();
- }
-
- @VisibleForTesting
- public long getNumDirectBlockWrites() {
- return numDirectBlockWrites.value();
- }
-
- @VisibleForTesting
- public long getNumFailedDirectBlockWrites() {
- return numFailedDirectBlockWrites.value();
- }
-
- @VisibleForTesting
- public long getNumFailedReadBlocks() {
- return numFailedReadBlocks.value();
- }
-
- @VisibleForTesting
- public long getNumWriteIOExceptionRetryBlocks() {
- return numWriteIOExceptionRetryBlocks.value();
- }
-
- @VisibleForTesting
- public long getNumWriteGenericExceptionRetryBlocks() {
- return numWriteGenericExceptionRetryBlocks.value();
- }
-
- @VisibleForTesting
- public long getNumBlockBufferFlushCompleted() {
- return numBlockBufferFlushCompleted.value();
- }
-
- @VisibleForTesting
- public long getNumBlockBufferFlushTriggered() {
- return numBlockBufferFlushTriggered.value();
- }
-
- @VisibleForTesting
- public long getNumDirtyLogBlockRead() {
- return numDirtyLogBlockRead.value();
- }
-
- @VisibleForTesting
- public long getNumBytesDirtyLogReads() {
- return numBytesDirtyLogRead.value();
- }
-
- @VisibleForTesting
- public long getNumBlockBufferUpdates() {
- return numBlockBufferUpdates.value();
- }
-
- @VisibleForTesting
- public long getNumRetryLogBlockRead() {
- return numRetryLogBlockRead.value();
- }
-
- @VisibleForTesting
- public long getNumBytesRetryLogReads() {
- return numBytesRetryLogRead.value();
- }
-
- @VisibleForTesting
- public long getNumBytesDirtyLogWritten() {
- return numBytesDirtyLogWritten.value();
- }
-
- @VisibleForTesting
- public long getNumFailedBlockBufferFlushes() {
- return numFailedBlockBufferFlushes.value();
- }
-
- @VisibleForTesting
- public long getNumInterruptedBufferWaits() {
- return numInterruptedBufferWaits.value();
- }
-
- @VisibleForTesting
- public long getNumIllegalDirtyLogFiles() {
- return numIllegalDirtyLogFiles.value();
- }
-
- @VisibleForTesting
- public long getNumFailedDirtyLogFileDeletes() {
- return numFailedDirtyLogFileDeletes.value();
- }
-
- @VisibleForTesting
- public long getNumFailedRetryLogFileWrites() {
- return numFailedRetryLogFileWrites.value();
- }
-
- @VisibleForTesting
- public long getNumWriteMaxRetryBlocks() {
- return numWriteMaxRetryBlocks.value();
- }
-
- @VisibleForTesting
- public long getNumFailedReleaseLevelDB() {
- return numFailedReleaseLevelDB.value();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
deleted file mode 100644
index 6c5c564..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/cblock/jscsiHelper/CBlockTargetServer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.cblock.jscsiHelper;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.cblock.proto.MountVolumeResponse;
-import org.apache.hadoop.cblock.util.KeyUtil;
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.scm.XceiverClientManager;
-import org.jscsi.target.Configuration;
-import org.jscsi.target.Target;
-import org.jscsi.target.TargetServer;
-
-import java.io.IOException;
-import java.util.HashMap;
-
-/**
- * This class extends JSCSI target server, which is a ISCSI target that can be
- * recognized by a remote machine with ISCSI installed.
- */
-public final class CBlockTargetServer extends TargetServer {
- private final OzoneConfiguration conf;
- private final CBlockManagerHandler cBlockManagerHandler;
- private final XceiverClientManager xceiverClientManager;
- private final ContainerCacheFlusher containerCacheFlusher;
- private final CBlockTargetMetrics metrics;
-
- public CBlockTargetServer(OzoneConfiguration ozoneConfig,
- Configuration jscsiConf,
- CBlockManagerHandler cBlockManagerHandler,
- CBlockTargetMetrics metrics)
- throws IOException {
- super(jscsiConf);
- this.cBlockManagerHandler = cBlockManagerHandler;
- this.xceiverClientManager = new XceiverClientManager(ozoneConfig);
- this.conf = ozoneConfig;
- this.containerCacheFlusher = new ContainerCacheFlusher(this.conf,
- xceiverClientManager, metrics);
- this.metrics = metrics;
- LOGGER.info("Starting flusher thread.");
- Thread flushListenerThread = new Thread(containerCacheFlusher);
- flushListenerThread.setDaemon(true);
- flushListenerThread.start();
- }
-
- public static void main(String[] args) throws Exception {
- }
-
- @Override
- public boolean isValidTargetName(String checkTargetName) {
- if (!KeyUtil.isValidVolumeKey(checkTargetName)) {
- return false;
- }
- String userName = KeyUtil.getUserNameFromVolumeKey(checkTargetName);
- String volumeName = KeyUtil.getVolumeFromVolumeKey(checkTargetName);
- if (userName == null || volumeName == null) {
- return false;
- }
- try {
- MountVolumeResponse result =
- cBlockManagerHandler.mountVolume(userName, volumeName);
- if (!result.getIsValid()) {
- LOGGER.error("Not a valid volume:" + checkTargetName);
- return false;
- }
- String volumeKey = KeyUtil.getVolumeKey(result.getUserName(),
- result.getVolumeName());
- if (!targets.containsKey(volumeKey)) {
- LOGGER.info("Mounting Volume. username: {} volume:{}",
- userName, volumeName);
- CBlockIStorageImpl ozoneStore = CBlockIStorageImpl.newBuilder()
- .setUserName(userName)
- .setVolumeName(volumeName)
- .setVolumeSize(result.getVolumeSize())
- .setBlockSize(result.getBlockSize())
- .setContainerList(result.getContainerList())
- .setClientManager(xceiverClientManager)
- .setConf(this.conf)
- .setFlusher(containerCacheFlusher)
- .setCBlockTargetMetrics(metrics)
- .build();
- Target target = new Target(volumeKey, volumeKey, ozoneStore);
- targets.put(volumeKey, target);
- }
- } catch (IOException e) {
- LOGGER.error("Can not connect to server when validating target!"
- + e.getMessage());
- }
- return targets.containsKey(checkTargetName);
- }
-
- @Override
- public String[] getTargetNames() {
- try {
- if (cBlockManagerHandler != null) {
- return cBlockManagerHandler.listVolumes().
- stream().map(
- volumeInfo -> volumeInfo.getUserName() + ":" + volumeInfo
- .getVolumeName()).toArray(String[]::new);
- } else {
- return new String[0];
- }
- } catch (IOException e) {
- LOGGER.error("Can't list existing volumes", e);
- return new String[0];
- }
- }
-
- @VisibleForTesting
- public HashMap<String, Target> getTargets() {
- return targets;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 6321923..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/KeySpaceManagerProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.common.collect.Lists;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmKeyLocationInfo;
-import org.apache.hadoop.ozone.ksm.helpers.KsmVolumeArgs;
-import org.apache.hadoop.ozone.ksm.helpers.OpenKeySession;
-import org.apache.hadoop.ozone.ksm.helpers.ServiceInfo;
-import org.apache.hadoop.ozone.ksm.protocol.KeySpaceManagerProtocol;
-import org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB;
-import org.apache.hadoop.ozone.ksm.exceptions.KSMException;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.AllocateBlockRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.AllocateBlockResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CommitKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CommitKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetBucketPropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetBucketPropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteBucketRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteBucketResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CreateVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.LocateKeyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.LocateKeyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.KeyArgs;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetVolumePropertyRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.SetVolumePropertyResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CheckVolumeAccessRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.CheckVolumeAccessResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.InfoVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.DeleteVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListVolumeRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListVolumeResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListBucketsRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListBucketsResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListKeysRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ListKeysResponse;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.Status;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServiceListRequest;
-import org.apache.hadoop.ozone.protocol.proto
- .KeySpaceManagerProtocolProtos.ServiceListResponse;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link org.apache.hadoop.ozone.ksm.protocolPB.KeySpaceManagerProtocolPB}
- * to the KeySpaceManagerService server implementation.
- */
-public class KeySpaceManagerProtocolServerSideTranslatorPB implements
- KeySpaceManagerProtocolPB {
- private static final Logger LOG = LoggerFactory
- .getLogger(KeySpaceManagerProtocolServerSideTranslatorPB.class);
- private final KeySpaceManagerProtocol impl;
-
- /**
- * Constructs an instance of the server handler.
- *
- * @param impl KeySpaceManagerProtocolPB
- */
- public KeySpaceManagerProtocolServerSideTranslatorPB(
- KeySpaceManagerProtocol impl) {
- this.impl = impl;
- }
-
- // Convert and exception to corresponding status code
- private Status exceptionToResponseStatus(IOException ex) {
- if (ex instanceof KSMException) {
- KSMException ksmException = (KSMException)ex;
- switch (ksmException.getResult()) {
- case FAILED_VOLUME_ALREADY_EXISTS:
- return Status.VOLUME_ALREADY_EXISTS;
- case FAILED_TOO_MANY_USER_VOLUMES:
- return Status.USER_TOO_MANY_VOLUMES;
- case FAILED_VOLUME_NOT_FOUND:
- return Status.VOLUME_NOT_FOUND;
- case FAILED_VOLUME_NOT_EMPTY:
- return Status.VOLUME_NOT_EMPTY;
- case FAILED_USER_NOT_FOUND:
- return Status.USER_NOT_FOUND;
- case FAILED_BUCKET_ALREADY_EXISTS:
- return Status.BUCKET_ALREADY_EXISTS;
- case FAILED_BUCKET_NOT_FOUND:
- return Status.BUCKET_NOT_FOUND;
- case FAILED_BUCKET_NOT_EMPTY:
- return Status.BUCKET_NOT_EMPTY;
- case FAILED_KEY_ALREADY_EXISTS:
- return Status.KEY_ALREADY_EXISTS;
- case FAILED_KEY_NOT_FOUND:
- return Status.KEY_NOT_FOUND;
- default:
- return Status.INTERNAL_ERROR;
- }
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Unknown error occurs", ex);
- }
- return Status.INTERNAL_ERROR;
- }
- }
-
- @Override
- public CreateVolumeResponse createVolume(
- RpcController controller, CreateVolumeRequest request)
- throws ServiceException {
- CreateVolumeResponse.Builder resp = CreateVolumeResponse.newBuilder();
- resp.setStatus(Status.OK);
- try {
- impl.createVolume(KsmVolumeArgs.getFromProtobuf(request.getVolumeInfo()));
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public SetVolumePropertyResponse setVolumeProperty(
- RpcController controller, SetVolumePropertyRequest request)
- throws ServiceException {
- SetVolumePropertyResponse.Builder resp =
- SetVolumePropertyResponse.newBuilder();
- resp.setStatus(Status.OK);
- String volume = request.getVolumeName();
-
- try {
- if (request.hasQuotaInBytes()) {
- long quota = request.getQuotaInBytes();
- impl.setQuota(volume, quota);
- } else {
- String owner = request.getOwnerName();
- impl.setOwner(volume, owner);
- }
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public CheckVolumeAccessResponse checkVolumeAccess(
- RpcController controller, CheckVolumeAccessRequest request)
- throws ServiceException {
- CheckVolumeAccessResponse.Builder resp =
- CheckVolumeAccessResponse.newBuilder();
- resp.setStatus(Status.OK);
- try {
- boolean access = impl.checkVolumeAccess(request.getVolumeName(),
- request.getUserAcl());
- // if no access, set the response status as access denied
- if (!access) {
- resp.setStatus(Status.ACCESS_DENIED);
- }
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
-
- return resp.build();
- }
-
- @Override
- public InfoVolumeResponse infoVolume(
- RpcController controller, InfoVolumeRequest request)
- throws ServiceException {
- InfoVolumeResponse.Builder resp = InfoVolumeResponse.newBuilder();
- resp.setStatus(Status.OK);
- String volume = request.getVolumeName();
- try {
- KsmVolumeArgs ret = impl.getVolumeInfo(volume);
- resp.setVolumeInfo(ret.getProtobuf());
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public DeleteVolumeResponse deleteVolume(
- RpcController controller, DeleteVolumeRequest request)
- throws ServiceException {
- DeleteVolumeResponse.Builder resp = DeleteVolumeResponse.newBuilder();
- resp.setStatus(Status.OK);
- try {
- impl.deleteVolume(request.getVolumeName());
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public ListVolumeResponse listVolumes(
- RpcController controller, ListVolumeRequest request)
- throws ServiceException {
- ListVolumeResponse.Builder resp = ListVolumeResponse.newBuilder();
- List<KsmVolumeArgs> result = Lists.newArrayList();
- try {
- if (request.getScope()
- == ListVolumeRequest.Scope.VOLUMES_BY_USER) {
- result = impl.listVolumeByUser(request.getUserName(),
- request.getPrefix(), request.getPrevKey(), request.getMaxKeys());
- } else if (request.getScope()
- == ListVolumeRequest.Scope.VOLUMES_BY_CLUSTER) {
- result = impl.listAllVolumes(request.getPrefix(), request.getPrevKey(),
- request.getMaxKeys());
- }
-
- if (result == null) {
- throw new ServiceException("Failed to get volumes for given scope "
- + request.getScope());
- }
-
- result.forEach(item -> resp.addVolumeInfo(item.getProtobuf()));
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public CreateBucketResponse createBucket(
- RpcController controller, CreateBucketRequest
- request) throws ServiceException {
- CreateBucketResponse.Builder resp =
- CreateBucketResponse.newBuilder();
- try {
- impl.createBucket(KsmBucketInfo.getFromProtobuf(
- request.getBucketInfo()));
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public InfoBucketResponse infoBucket(
- RpcController controller, InfoBucketRequest request)
- throws ServiceException {
- InfoBucketResponse.Builder resp =
- InfoBucketResponse.newBuilder();
- try {
- KsmBucketInfo ksmBucketInfo = impl.getBucketInfo(
- request.getVolumeName(), request.getBucketName());
- resp.setStatus(Status.OK);
- resp.setBucketInfo(ksmBucketInfo.getProtobuf());
- } catch(IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public LocateKeyResponse createKey(
- RpcController controller, LocateKeyRequest request
- ) throws ServiceException {
- LocateKeyResponse.Builder resp =
- LocateKeyResponse.newBuilder();
- try {
- KeyArgs keyArgs = request.getKeyArgs();
- OzoneProtos.ReplicationType type =
- keyArgs.hasType()? keyArgs.getType() : null;
- OzoneProtos.ReplicationFactor factor =
- keyArgs.hasFactor()? keyArgs.getFactor() : null;
- KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .setDataSize(keyArgs.getDataSize())
- .setType(type)
- .setFactor(factor)
- .build();
- if (keyArgs.hasDataSize()) {
- ksmKeyArgs.setDataSize(keyArgs.getDataSize());
- } else {
- ksmKeyArgs.setDataSize(0);
- }
- OpenKeySession openKey = impl.openKey(ksmKeyArgs);
- resp.setKeyInfo(openKey.getKeyInfo().getProtobuf());
- resp.setID(openKey.getId());
- resp.setOpenVersion(openKey.getOpenVersion());
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public LocateKeyResponse lookupKey(
- RpcController controller, LocateKeyRequest request
- ) throws ServiceException {
- LocateKeyResponse.Builder resp =
- LocateKeyResponse.newBuilder();
- try {
- KeyArgs keyArgs = request.getKeyArgs();
- KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .build();
- KsmKeyInfo keyInfo = impl.lookupKey(ksmKeyArgs);
- resp.setKeyInfo(keyInfo.getProtobuf());
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public SetBucketPropertyResponse setBucketProperty(
- RpcController controller, SetBucketPropertyRequest request)
- throws ServiceException {
- SetBucketPropertyResponse.Builder resp =
- SetBucketPropertyResponse.newBuilder();
- try {
- impl.setBucketProperty(KsmBucketArgs.getFromProtobuf(
- request.getBucketArgs()));
- resp.setStatus(Status.OK);
- } catch(IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public LocateKeyResponse deleteKey(RpcController controller,
- LocateKeyRequest request) throws ServiceException {
- LocateKeyResponse.Builder resp =
- LocateKeyResponse.newBuilder();
- try {
- KeyArgs keyArgs = request.getKeyArgs();
- KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .build();
- impl.deleteKey(ksmKeyArgs);
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public DeleteBucketResponse deleteBucket(
- RpcController controller, DeleteBucketRequest request)
- throws ServiceException {
- DeleteBucketResponse.Builder resp = DeleteBucketResponse.newBuilder();
- resp.setStatus(Status.OK);
- try {
- impl.deleteBucket(request.getVolumeName(), request.getBucketName());
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public ListBucketsResponse listBuckets(
- RpcController controller, ListBucketsRequest request)
- throws ServiceException {
- ListBucketsResponse.Builder resp =
- ListBucketsResponse.newBuilder();
- try {
- List<KsmBucketInfo> buckets = impl.listBuckets(
- request.getVolumeName(),
- request.getStartKey(),
- request.getPrefix(),
- request.getCount());
- for(KsmBucketInfo bucket : buckets) {
- resp.addBucketInfo(bucket.getProtobuf());
- }
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public ListKeysResponse listKeys(RpcController controller,
- ListKeysRequest request) throws ServiceException {
- ListKeysResponse.Builder resp =
- ListKeysResponse.newBuilder();
- try {
- List<KsmKeyInfo> keys = impl.listKeys(
- request.getVolumeName(),
- request.getBucketName(),
- request.getStartKey(),
- request.getPrefix(),
- request.getCount());
- for(KsmKeyInfo key : keys) {
- resp.addKeyInfo(key.getProtobuf());
- }
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public CommitKeyResponse commitKey(RpcController controller,
- CommitKeyRequest request) throws ServiceException {
- CommitKeyResponse.Builder resp =
- CommitKeyResponse.newBuilder();
- try {
- KeyArgs keyArgs = request.getKeyArgs();
- OzoneProtos.ReplicationType type =
- keyArgs.hasType()? keyArgs.getType() : null;
- OzoneProtos.ReplicationFactor factor =
- keyArgs.hasFactor()? keyArgs.getFactor() : null;
- KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .setDataSize(keyArgs.getDataSize())
- .setType(type)
- .setFactor(factor)
- .build();
- int id = request.getClientID();
- impl.commitKey(ksmKeyArgs, id);
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public AllocateBlockResponse allocateBlock(RpcController controller,
- AllocateBlockRequest request) throws ServiceException {
- AllocateBlockResponse.Builder resp =
- AllocateBlockResponse.newBuilder();
- try {
- KeyArgs keyArgs = request.getKeyArgs();
- OzoneProtos.ReplicationType type =
- keyArgs.hasType()? keyArgs.getType() : null;
- OzoneProtos.ReplicationFactor factor =
- keyArgs.hasFactor()? keyArgs.getFactor() : null;
- KsmKeyArgs ksmKeyArgs = new KsmKeyArgs.Builder()
- .setVolumeName(keyArgs.getVolumeName())
- .setBucketName(keyArgs.getBucketName())
- .setKeyName(keyArgs.getKeyName())
- .setType(type)
- .setFactor(factor)
- .build();
- int id = request.getClientID();
- KsmKeyLocationInfo newLocation = impl.allocateBlock(ksmKeyArgs, id);
- resp.setKeyLocation(newLocation.getProtobuf());
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-
- @Override
- public ServiceListResponse getServiceList(RpcController controller,
- ServiceListRequest request) throws ServiceException {
- ServiceListResponse.Builder resp = ServiceListResponse.newBuilder();
- try {
- resp.addAllServiceInfo(impl.getServiceList().stream()
- .map(ServiceInfo::getProtobuf)
- .collect(Collectors.toList()));
- resp.setStatus(Status.OK);
- } catch (IOException e) {
- resp.setStatus(exceptionToResponseStatus(e));
- }
- return resp.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 2ab11ba..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/ScmBlockLocationProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.common.collect.Sets;
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.ScmBlockLocationProtocolProtos.DeleteKeyBlocksResultProto;
-import org.apache.hadoop.scm.container.common.helpers.AllocatedBlock;
-import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.protocol.ScmBlockLocationProtocol;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-import org.apache.hadoop.scm.protocolPB.ScmBlockLocationProtocolPB;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.AllocateScmBlockRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.AllocateScmBlockResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.DeleteScmKeyBlocksResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.GetScmBlockLocationsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.GetScmBlockLocationsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .ScmBlockLocationProtocolProtos.ScmLocatedBlockProto;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerLocationProtocolPB} to the
- * {@link StorageContainerLocationProtocol} server implementation.
- */
-@InterfaceAudience.Private
-public final class ScmBlockLocationProtocolServerSideTranslatorPB
- implements ScmBlockLocationProtocolPB {
-
- private final ScmBlockLocationProtocol impl;
-
- /**
- * Creates a new ScmBlockLocationProtocolServerSideTranslatorPB.
- *
- * @param impl {@link ScmBlockLocationProtocol} server implementation
- */
- public ScmBlockLocationProtocolServerSideTranslatorPB(
- ScmBlockLocationProtocol impl) throws IOException {
- this.impl = impl;
- }
-
-
- @Override
- public GetScmBlockLocationsResponseProto getScmBlockLocations(
- RpcController controller, GetScmBlockLocationsRequestProto req)
- throws ServiceException {
- Set<String> keys = Sets.newLinkedHashSetWithExpectedSize(
- req.getKeysCount());
- for (String key : req.getKeysList()) {
- keys.add(key);
- }
- final Set<AllocatedBlock> blocks;
- try {
- blocks = impl.getBlockLocations(keys);
- } catch (IOException ex) {
- throw new ServiceException(ex);
- }
- GetScmBlockLocationsResponseProto.Builder resp =
- GetScmBlockLocationsResponseProto.newBuilder();
- for (AllocatedBlock block: blocks) {
- ScmLocatedBlockProto.Builder locatedBlock =
- ScmLocatedBlockProto.newBuilder()
- .setKey(block.getKey())
- .setPipeline(block.getPipeline().getProtobufMessage());
- resp.addLocatedBlocks(locatedBlock.build());
- }
- return resp.build();
- }
-
- @Override
- public AllocateScmBlockResponseProto allocateScmBlock(
- RpcController controller, AllocateScmBlockRequestProto request)
- throws ServiceException {
- try {
- AllocatedBlock allocatedBlock =
- impl.allocateBlock(request.getSize(), request.getType(),
- request.getFactor(), request.getOwner());
- if (allocatedBlock != null) {
- return
- AllocateScmBlockResponseProto.newBuilder()
- .setKey(allocatedBlock.getKey())
- .setPipeline(allocatedBlock.getPipeline().getProtobufMessage())
- .setCreateContainer(allocatedBlock.getCreateContainer())
- .setErrorCode(AllocateScmBlockResponseProto.Error.success)
- .build();
- } else {
- return AllocateScmBlockResponseProto.newBuilder()
- .setErrorCode(AllocateScmBlockResponseProto.Error.unknownFailure)
- .build();
- }
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public DeleteScmKeyBlocksResponseProto deleteScmKeyBlocks(
- RpcController controller, DeleteScmKeyBlocksRequestProto req)
- throws ServiceException {
- DeleteScmKeyBlocksResponseProto.Builder resp =
- DeleteScmKeyBlocksResponseProto.newBuilder();
- try {
- List<BlockGroup> infoList = req.getKeyBlocksList().stream()
- .map(BlockGroup::getFromProto).collect(Collectors.toList());
- final List<DeleteBlockGroupResult> results =
- impl.deleteKeyBlocks(infoList);
- for (DeleteBlockGroupResult result: results) {
- DeleteKeyBlocksResultProto.Builder deleteResult =
- DeleteKeyBlocksResultProto
- .newBuilder()
- .setObjectKey(result.getObjectKey())
- .addAllBlockResults(result.getBlockResultProtoList());
- resp.addResults(deleteResult.build());
- }
- } catch (IOException ex) {
- throw new ServiceException(ex);
- }
- return resp.build();
- }
-
- @Override
- public OzoneProtos.GetScmInfoRespsonseProto getScmInfo(
- RpcController controller, OzoneProtos.GetScmInfoRequestProto req)
- throws ServiceException {
- ScmInfo scmInfo;
- try {
- scmInfo = impl.getScmInfo();
- } catch (IOException ex) {
- throw new ServiceException(ex);
- }
- return OzoneProtos.GetScmInfoRespsonseProto.newBuilder()
- .setClusterId(scmInfo.getClusterId())
- .setScmId(scmInfo.getScmId())
- .build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
deleted file mode 100644
index 50d4fbb..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolClientSideTranslatorPB.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ipc.ProtobufHelper;
-import org.apache.hadoop.ipc.ProtocolTranslator;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMRegisterRequestProto;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-
-import java.io.Closeable;
-import java.io.IOException;
-
-/**
- * This class is the client-side translator to translate the requests made on
- * the {@link StorageContainerDatanodeProtocol} interface to the RPC server
- * implementing {@link StorageContainerDatanodeProtocolPB}.
- */
-public class StorageContainerDatanodeProtocolClientSideTranslatorPB
- implements StorageContainerDatanodeProtocol, ProtocolTranslator, Closeable {
-
- /**
- * RpcController is not used and hence is set to null.
- */
- private static final RpcController NULL_RPC_CONTROLLER = null;
- private final StorageContainerDatanodeProtocolPB rpcProxy;
-
- /**
- * Constructs a Client side interface that calls into SCM datanode protocol.
- *
- * @param rpcProxy - Proxy for RPC.
- */
- public StorageContainerDatanodeProtocolClientSideTranslatorPB(
- StorageContainerDatanodeProtocolPB rpcProxy) {
- this.rpcProxy = rpcProxy;
- }
-
- /**
- * Closes this stream and releases any system resources associated with it. If
- * the stream is already closed then invoking this method has no effect.
- * <p>
- * <p> As noted in {@link AutoCloseable#close()}, cases where the close may
- * fail require careful attention. It is strongly advised to relinquish the
- * underlying resources and to internally <em>mark</em> the {@code Closeable}
- * as closed, prior to throwing the {@code IOException}.
- *
- * @throws IOException if an I/O error occurs
- */
- @Override
- public void close() throws IOException {
- RPC.stopProxy(rpcProxy);
- }
-
- /**
- * Return the proxy object underlying this protocol translator.
- *
- * @return the proxy object underlying this protocol translator.
- */
- @Override
- public Object getUnderlyingProxyObject() {
- return rpcProxy;
- }
-
- /**
- * Returns SCM version.
- *
- * @param unused - set to null and unused.
- * @return Version info.
- */
- @Override
- public SCMVersionResponseProto getVersion(SCMVersionRequestProto
- unused) throws IOException {
- SCMVersionRequestProto request =
- SCMVersionRequestProto.newBuilder().build();
- final SCMVersionResponseProto response;
- try {
- response = rpcProxy.getVersion(NULL_RPC_CONTROLLER, request);
- } catch (ServiceException ex) {
- throw ProtobufHelper.getRemoteException(ex);
- }
- return response;
- }
-
- /**
- * Send by datanode to SCM.
- *
- * @param datanodeID - DatanodeID
- * @param nodeReport - node report
- * @throws IOException
- */
-
- @Override
- public SCMHeartbeatResponseProto sendHeartbeat(DatanodeID datanodeID,
- SCMNodeReport nodeReport, ReportState reportState) throws IOException {
- SCMHeartbeatRequestProto.Builder req = SCMHeartbeatRequestProto
- .newBuilder();
- req.setDatanodeID(datanodeID.getProtoBufMessage());
- req.setNodeReport(nodeReport);
- req.setContainerReportState(reportState);
- final SCMHeartbeatResponseProto resp;
- try {
- resp = rpcProxy.sendHeartbeat(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- return resp;
- }
-
- /**
- * Register Datanode.
- *
- * @param datanodeID - DatanodID.
- * @return SCM Command.
- */
- @Override
- public SCMRegisteredCmdResponseProto register(DatanodeID datanodeID,
- String[] scmAddresses) throws IOException {
- SCMRegisterRequestProto.Builder req =
- SCMRegisterRequestProto.newBuilder();
- req.setDatanodeID(datanodeID.getProtoBufMessage());
- final SCMRegisteredCmdResponseProto response;
- try {
- response = rpcProxy.register(NULL_RPC_CONTROLLER, req.build());
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- return response;
- }
-
- /**
- * Send a container report.
- *
- * @param reports -- Container report
- * @return HeartbeatRespose.nullcommand.
- * @throws IOException
- */
- @Override
- public ContainerReportsResponseProto sendContainerReport(
- ContainerReportsRequestProto reports) throws IOException {
- final ContainerReportsResponseProto resp;
- try {
- resp = rpcProxy.sendContainerReport(NULL_RPC_CONTROLLER, reports);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- return resp;
- }
-
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- ContainerBlocksDeletionACKProto deletedBlocks) throws IOException {
- final ContainerBlocksDeletionACKResponseProto resp;
- try {
- resp = rpcProxy.sendContainerBlocksDeletionACK(NULL_RPC_CONTROLLER,
- deletedBlocks);
- } catch (ServiceException e) {
- throw ProtobufHelper.getRemoteException(e);
- }
- return resp;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
deleted file mode 100644
index 3e6f392..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolPB.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.StorageContainerDatanodeProtocolService;
-
-/**
- * Protocol used from a datanode to StorageContainerManager. This extends
- * the Protocol Buffers service interface to add Hadoop-specific annotations.
- */
-
-@ProtocolInfo(protocolName =
- "org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol",
- protocolVersion = 1)
-public interface StorageContainerDatanodeProtocolPB extends
- StorageContainerDatanodeProtocolService.BlockingInterface {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
deleted file mode 100644
index 2fd7038..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerDatanodeProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.protocol.StorageContainerDatanodeProtocol;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMHeartbeatResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerBlocksDeletionACKProto;
-
-import java.io.IOException;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerDatanodeProtocolPB} to the {@link
- * StorageContainerDatanodeProtocol} server implementation.
- */
-public class StorageContainerDatanodeProtocolServerSideTranslatorPB
- implements StorageContainerDatanodeProtocolPB {
-
- private final StorageContainerDatanodeProtocol impl;
-
- public StorageContainerDatanodeProtocolServerSideTranslatorPB(
- StorageContainerDatanodeProtocol impl) {
- this.impl = impl;
- }
-
- @Override
- public StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto
- getVersion(RpcController controller,
- StorageContainerDatanodeProtocolProtos.SCMVersionRequestProto request)
- throws ServiceException {
- try {
- return impl.getVersion(request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public StorageContainerDatanodeProtocolProtos.SCMRegisteredCmdResponseProto
- register(RpcController controller, StorageContainerDatanodeProtocolProtos
- .SCMRegisterRequestProto request) throws ServiceException {
- String[] addressArray = null;
-
- if (request.hasAddressList()) {
- addressArray = request.getAddressList().getAddressListList()
- .toArray(new String[0]);
- }
-
- try {
- return impl.register(DatanodeID.getFromProtoBuf(request
- .getDatanodeID()), addressArray);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public SCMHeartbeatResponseProto
- sendHeartbeat(RpcController controller,
- SCMHeartbeatRequestProto request) throws ServiceException {
- try {
- return impl.sendHeartbeat(DatanodeID.getFromProtoBuf(request
- .getDatanodeID()), request.getNodeReport(),
- request.getContainerReportState());
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public ContainerReportsResponseProto sendContainerReport(
- RpcController controller, ContainerReportsRequestProto request)
- throws ServiceException {
- try {
- return impl.sendContainerReport(request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public ContainerBlocksDeletionACKResponseProto sendContainerBlocksDeletionACK(
- RpcController controller, ContainerBlocksDeletionACKProto request)
- throws ServiceException {
- try {
- return impl.sendContainerBlocksDeletionACK(request);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
deleted file mode 100644
index c8258b9..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/StorageContainerLocationProtocolServerSideTranslatorPB.java
+++ /dev/null
@@ -1,202 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.protocolPB;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.List;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerLocationProtocolProtos;
-import org.apache.hadoop.scm.ScmInfo;
-import org.apache.hadoop.scm.container.common.helpers.ContainerInfo;
-import org.apache.hadoop.scm.protocol.StorageContainerLocationProtocol;
-
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.GetContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.DeleteContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ListContainerRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.ObjectStageChangeResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineResponseProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerLocationProtocolProtos.PipelineRequestProto;
-import org.apache.hadoop.scm.container.common.helpers.Pipeline;
-import org.apache.hadoop.scm.protocolPB.StorageContainerLocationProtocolPB;
-
-/**
- * This class is the server-side translator that forwards requests received on
- * {@link StorageContainerLocationProtocolPB} to the
- * {@link StorageContainerLocationProtocol} server implementation.
- */
-@InterfaceAudience.Private
-public final class StorageContainerLocationProtocolServerSideTranslatorPB
- implements StorageContainerLocationProtocolPB {
-
- private final StorageContainerLocationProtocol impl;
-
- /**
- * Creates a new StorageContainerLocationProtocolServerSideTranslatorPB.
- *
- * @param impl {@link StorageContainerLocationProtocol} server implementation
- */
- public StorageContainerLocationProtocolServerSideTranslatorPB(
- StorageContainerLocationProtocol impl) throws IOException {
- this.impl = impl;
- }
-
- @Override
- public ContainerResponseProto allocateContainer(RpcController unused,
- ContainerRequestProto request) throws ServiceException {
- try {
- Pipeline pipeline = impl.allocateContainer(request.getReplicationType(),
- request.getReplicationFactor(), request.getContainerName(),
- request.getOwner());
- return ContainerResponseProto.newBuilder()
- .setPipeline(pipeline.getProtobufMessage())
- .setErrorCode(ContainerResponseProto.Error.success)
- .build();
-
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public GetContainerResponseProto getContainer(
- RpcController controller, GetContainerRequestProto request)
- throws ServiceException {
- try {
- Pipeline pipeline = impl.getContainer(request.getContainerName());
- return GetContainerResponseProto.newBuilder()
- .setPipeline(pipeline.getProtobufMessage())
- .build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public ListContainerResponseProto listContainer(RpcController controller,
- ListContainerRequestProto request) throws ServiceException {
- try {
- String startName = null;
- String prefixName = null;
- int count = -1;
-
- // Arguments check.
- if (request.hasPrefixName()) {
- // End container name is given.
- prefixName = request.getPrefixName();
- }
- if (request.hasStartName()) {
- // End container name is given.
- startName = request.getStartName();
- }
-
- count = request.getCount();
- List<ContainerInfo> containerList =
- impl.listContainer(startName, prefixName, count);
- ListContainerResponseProto.Builder builder =
- ListContainerResponseProto.newBuilder();
- for (ContainerInfo container : containerList) {
- builder.addContainers(container.getProtobuf());
- }
- return builder.build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public DeleteContainerResponseProto deleteContainer(
- RpcController controller, DeleteContainerRequestProto request)
- throws ServiceException {
- try {
- impl.deleteContainer(request.getContainerName());
- return DeleteContainerResponseProto.newBuilder().build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public StorageContainerLocationProtocolProtos.NodeQueryResponseProto
- queryNode(RpcController controller,
- StorageContainerLocationProtocolProtos.NodeQueryRequestProto request)
- throws ServiceException {
- try {
- EnumSet<OzoneProtos.NodeState> nodeStateEnumSet = EnumSet.copyOf(request
- .getQueryList());
- OzoneProtos.NodePool datanodes = impl.queryNode(nodeStateEnumSet,
- request.getScope(), request.getPoolName());
- return StorageContainerLocationProtocolProtos
- .NodeQueryResponseProto.newBuilder()
- .setDatanodes(datanodes)
- .build();
- } catch (Exception e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public ObjectStageChangeResponseProto notifyObjectStageChange(
- RpcController controller, ObjectStageChangeRequestProto request)
- throws ServiceException {
- try {
- impl.notifyObjectStageChange(request.getType(), request.getName(),
- request.getOp(), request.getStage());
- return ObjectStageChangeResponseProto.newBuilder().build();
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public PipelineResponseProto allocatePipeline(
- RpcController controller, PipelineRequestProto request)
- throws ServiceException {
- // TODO : Wiring this up requires one more patch.
- return null;
- }
-
- @Override
- public OzoneProtos.GetScmInfoRespsonseProto getScmInfo(
- RpcController controller, OzoneProtos.GetScmInfoRequestProto req)
- throws ServiceException {
- try {
- ScmInfo scmInfo = impl.getScmInfo();
- return OzoneProtos.GetScmInfoRespsonseProto.newBuilder()
- .setClusterId(scmInfo.getClusterId())
- .setScmId(scmInfo.getScmId())
- .build();
- } catch (IOException ex) {
- throw new ServiceException(ex);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
deleted file mode 100644
index 860386d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/protocolPB/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.protocolPB;
-
-/**
- * This package contains classes for the Protocol Buffers binding of Ozone
- * protocols.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
deleted file mode 100644
index 336ed44..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMMXBean.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.scm;
-
-import java.util.Map;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.jmx.ServiceRuntimeInfo;
-
-/**
- *
- * This is the JMX management interface for scm information.
- */
-@InterfaceAudience.Private
-public interface SCMMXBean extends ServiceRuntimeInfo {
-
- /**
- * Get the SCM RPC server port that used to listen to datanode requests.
- * @return SCM datanode RPC server port
- */
- String getDatanodeRpcPort();
-
- /**
- * Get the SCM RPC server port that used to listen to client requests.
- * @return SCM client RPC server port
- */
- String getClientRpcPort();
-
- /**
- * Get container report info that includes container IO stats of nodes.
- * @return The datanodeUUid to report json string mapping
- */
- Map<String, String> getContainerReport();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
deleted file mode 100644
index e5437c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/scm/SCMStorage.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.scm;
-
-
-import org.apache.hadoop.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.common.Storage;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos.NodeType;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-
-import java.io.IOException;
-import java.util.Properties;
-import java.util.UUID;
-
-/**
- * SCMStorage is responsible for management of the StorageDirectories used by
- * the SCM.
- */
-public class SCMStorage extends Storage {
-
- public static final String STORAGE_DIR = "scm";
- public static final String SCM_ID = "scmUuid";
-
- /**
- * Construct SCMStorage.
- * @throws IOException if any directories are inaccessible.
- */
- public SCMStorage(OzoneConfiguration conf) throws IOException {
- super(NodeType.SCM, OzoneUtils.getOzoneMetaDirPath(conf), STORAGE_DIR);
- }
-
- public void setScmId(String scmId) throws IOException {
- if (getState() == StorageState.INITIALIZED) {
- throw new IOException("SCM is already initialized.");
- } else {
- getStorageInfo().setProperty(SCM_ID, scmId);
- }
- }
-
- /**
- * Retrieves the SCM ID from the version file.
- * @return SCM_ID
- */
- public String getScmId() {
- return getStorageInfo().getProperty(SCM_ID);
- }
-
- @Override
- protected Properties getNodeProperties() {
- String scmId = getScmId();
- if (scmId == null) {
- scmId = UUID.randomUUID().toString();
- }
- Properties scmProperties = new Properties();
- scmProperties.setProperty(SCM_ID, scmId);
- return scmProperties;
- }
-
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
deleted file mode 100644
index 0cd652a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/VolumeProcessTemplate.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.handlers;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.protocol.proto.KeySpaceManagerProtocolProtos;
-import org.apache.hadoop.ozone.web.exceptions.ErrorTable;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.web.interfaces.UserAuth;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-import org.apache.hadoop.ozone.web.utils.OzoneUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.MDC;
-
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.IOException;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.FileAlreadyExistsException;
-import java.nio.file.NoSuchFileException;
-
-import static java.net.HttpURLConnection.HTTP_OK;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_COMPONENT;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_RESOURCE;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_REQUEST;
-import static org.apache.hadoop.ozone.OzoneConsts.OZONE_USER;
-
-
-/**
- * This class abstracts way the repetitive tasks in
- * handling volume related code.
- */
-@InterfaceAudience.Private
-public abstract class VolumeProcessTemplate {
- private static final Logger LOG =
- LoggerFactory.getLogger(VolumeProcessTemplate.class);
-
-
- /**
- * The handle call is the common functionality for Volume
- * handling code.
- *
- * @param volume - Name of the Volume
- * @param request - request
- * @param info - UriInfo
- * @param headers - Http Headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- public Response handleCall(String volume, Request request, UriInfo info,
- HttpHeaders headers) throws OzoneException {
- String reqID = OzoneUtils.getRequestID();
- String hostName = OzoneUtils.getHostName();
- MDC.put(OZONE_COMPONENT, "ozone");
- MDC.put(OZONE_REQUEST, reqID);
- UserArgs userArgs = null;
- try {
- userArgs = new UserArgs(reqID, hostName, request, info, headers);
- OzoneUtils.validate(request, headers, reqID, volume, hostName);
-
- // we use the same logic for both bucket and volume names
- OzoneUtils.verifyResourceName(volume);
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
-
- userArgs.setUserName(auth.getUser(userArgs));
- MDC.put(OZONE_USER, userArgs.getUserName());
- VolumeArgs args = new VolumeArgs(volume, userArgs);
-
- MDC.put(OZONE_RESOURCE, args.getResourceName());
- Response response = doProcess(args);
- LOG.info("Success");
- MDC.clear();
- return response;
-
- } catch (IllegalArgumentException ex) {
- LOG.error("Illegal argument.", ex);
- throw ErrorTable.newError(ErrorTable.INVALID_VOLUME_NAME, userArgs, ex);
- } catch (IOException ex) {
- handleIOException(volume, reqID, hostName, ex);
- }
- return null;
- }
-
- /**
- * Specific handler for each call.
- *
- * @param args - Volume Args
- *
- * @return - Response
- *
- * @throws IOException
- * @throws OzoneException
- */
- public abstract Response doProcess(VolumeArgs args)
- throws IOException, OzoneException;
-
- /**
- * Maps Java File System Exceptions to Ozone Exceptions in the Volume path.
- *
- * @param volume - Name of the Volume
- * @param reqID - Request ID
- * @param hostName - HostName
- * @param fsExp - Exception
- *
- * @throws OzoneException
- */
- private void handleIOException(String volume, String reqID, String hostName,
- IOException fsExp) throws OzoneException {
- LOG.error("IOException:", fsExp);
- OzoneException exp = null;
-
- if ((fsExp != null && fsExp.getMessage().endsWith(
- KeySpaceManagerProtocolProtos.Status.VOLUME_ALREADY_EXISTS.name()))
- || fsExp instanceof FileAlreadyExistsException) {
- exp = ErrorTable
- .newError(ErrorTable.VOLUME_ALREADY_EXISTS, reqID, volume, hostName);
- }
-
- if (fsExp instanceof DirectoryNotEmptyException) {
- exp = ErrorTable
- .newError(ErrorTable.VOLUME_NOT_EMPTY, reqID, volume, hostName);
- }
-
- if (fsExp instanceof NoSuchFileException) {
- exp = ErrorTable
- .newError(ErrorTable.INVALID_VOLUME_NAME, reqID, volume, hostName);
- }
-
- if ((fsExp != null) && (exp != null)) {
- exp.setMessage(fsExp.getMessage());
- }
-
- // We don't handle that FS error yet, report a Server Internal Error
- if (exp == null) {
- exp =
- ErrorTable.newError(ErrorTable.SERVER_ERROR, reqID, volume, hostName);
- if (fsExp != null) {
- exp.setMessage(fsExp.getMessage());
- }
- }
- throw exp;
- }
-
- /**
- * Set the user provided string into args and throw ozone exception
- * if needed.
- *
- * @param args - volume args
- * @param quota - quota sting
- *
- * @throws OzoneException
- */
- void setQuotaArgs(VolumeArgs args, String quota) throws OzoneException {
- try {
- args.setQuota(quota);
- } catch (IllegalArgumentException ex) {
- LOG.debug("Malformed Quota.", ex);
- throw ErrorTable.newError(ErrorTable.MALFORMED_QUOTA, args, ex);
- }
- }
-
- /**
- * Wraps calls into volumeInfo data.
- *
- * @param args - volumeArgs
- *
- * @return - VolumeInfo
- *
- * @throws IOException
- * @throws OzoneException
- */
- Response getVolumeInfoResponse(VolumeArgs args)
- throws IOException, OzoneException {
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- VolumeInfo info = fs.getVolumeInfo(args);
- return OzoneUtils.getResponse(args, HTTP_OK, info.toJsonString());
- }
-
- /**
- * Returns all the volumes belonging to a user.
- *
- * @param user - userArgs
- * @return - Response
- * @throws OzoneException
- * @throws IOException
- */
- Response getVolumesByUser(UserArgs user, String prefix, int maxKeys,
- String prevKey, boolean rootScan) throws OzoneException, IOException {
-
- String validatedUser = user.getUserName();
- try {
- UserAuth auth = UserHandlerBuilder.getAuthHandler();
- if(rootScan && !auth.isAdmin(user)) {
- throw ErrorTable.newError(ErrorTable.UNAUTHORIZED, user);
- }
- if (auth.isAdmin(user)) {
- validatedUser = auth.getOzoneUser(user);
- if (validatedUser == null) {
- validatedUser = auth.getUser(user);
- }
- }
-
- UserArgs onBehalfOf =
- new UserArgs(validatedUser, user.getRequestID(), user.getHostName(),
- user.getRequest(), user.getUri(), user.getHeaders());
-
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- ListArgs<UserArgs> listArgs = new ListArgs<>(onBehalfOf, prefix,
- maxKeys, prevKey);
- listArgs.setRootScan(rootScan);
- ListVolumes volumes = fs.listVolumes(listArgs);
- return OzoneUtils.getResponse(user, HTTP_OK, volumes.toJsonString());
- } catch (IOException ex) {
- LOG.debug("unable to get the volume list for the user.", ex);
- OzoneException exp = ErrorTable.newError(ErrorTable.SERVER_ERROR,
- user, ex);
- exp.setMessage("unable to get the volume list for the user");
- throw exp;
- }
- }
-
- /**
- * Returns a list of Buckets in a Volume.
- *
- * @param args - VolumeArgs
- * @param prefix - Prefix to Match
- * @param maxKeys - Max results to return.
- * @param prevKey - PrevKey
- * @return List of Buckets
- * @throws OzoneException
- */
- Response getBucketsInVolume(VolumeArgs args, String prefix, int maxKeys,
- String prevKey) throws OzoneException {
- try {
- // UserAuth auth = UserHandlerBuilder.getAuthHandler();
- // TODO : Check ACLS.
- StorageHandler fs = StorageHandlerBuilder.getStorageHandler();
- ListArgs<VolumeArgs> listArgs = new ListArgs<>(args, prefix,
- maxKeys, prevKey);
- ListBuckets bucketList = fs.listBuckets(listArgs);
- return OzoneUtils.getResponse(args, HTTP_OK, bucketList.toJsonString());
- } catch (IOException ex) {
- LOG.debug("unable to get the bucket list for the specified volume.", ex);
- OzoneException exp =
- ErrorTable.newError(ErrorTable.SERVER_ERROR, args, ex);
- exp.setMessage("unable to get the bucket list for the specified volume.");
- throw exp;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
deleted file mode 100644
index 4d34c2d..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.handlers;
-
-/**
- This package contains ozone client side libraries.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
deleted file mode 100644
index f03276c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Accounting.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-/**
- * This in the accounting interface, Ozone Rest interface will call into this
- * interface whenever a put or delete key happens.
- * <p>
- * TODO : Technically we need to report bucket creation and deletion too
- * since the bucket names and metadata consume storage.
- * <p>
- * TODO : We should separate out reporting metadata & data --
- * <p>
- * In some cases end users will only want to account for the data they are
- * storing since metadata is mostly a cost of business.
- */
-public interface Accounting {
- /**
- * This call is made when ever a put key call is made.
- * <p>
- * In case of a Put which causes a over write of a key accounting system will
- * see two calls, a removeByte call followed by an addByte call.
- *
- * @param owner - Volume Owner
- * @param volume - Name of the Volume
- * @param bucket - Name of the bucket
- * @param bytes - How many bytes are put
- */
- void addBytes(String owner, String volume, String bucket, int bytes);
-
- /**
- * This call is made whenever a delete call is made.
- *
- * @param owner - Volume Owner
- * @param volume - Name of the Volume
- * @param bucket - Name of the bucket
- * @param bytes - How many bytes are deleted
- */
- void removeBytes(String owner, String volume, String bucket, int bytes);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
deleted file mode 100644
index 4c8b85b..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Bucket.java
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.Produces;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * Bucket Interface acts as the HTTP entry point for
- * bucket related functionality.
- */
-@Path("/{volume}/{bucket}")
-@Api(tags = "bucket")
-public interface Bucket {
- /**
- * createBucket call handles the POST request for Creating a Bucket.
- *
- * @param volume - Volume name
- * @param bucket - Bucket Name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @POST
- @ApiOperation("Create new bucket to a volume")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response createBucket(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket,
- @Context Request req, @Context UriInfo info,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * updateBucket call handles the PUT request for updating a Bucket.
- *
- * @param volume - Volume name
- * @param bucket - Bucket name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @PUT
- @ApiOperation("Update bucket")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response updateBucket(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket,
- @Context Request req, @Context UriInfo info,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Deletes an empty bucket.
- *
- * @param volume Volume name
- * @param bucket Bucket Name
- * @param req - Http request
- * @param info - Uri Info
- * @param headers - Http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @DELETE
- @ApiOperation("Deletes an empty bucket.")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response deleteBucket(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket,
- @Context Request req, @Context UriInfo info,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * List Buckets lists the contents of a bucket.
- *
- * @param volume - Storage Volume Name
- * @param bucket - Bucket Name
- * @param info - Information type needed
- * @param prefix - Prefix for the keys to be fetched
- * @param maxKeys - MaxNumber of Keys to Return
- * @param prevKey - Continuation Token
- * @param req - Http request
- * @param headers - Http headers
- *
- * @return - Json Body
- *
- * @throws OzoneException
- */
-
- @GET
- @Produces(MediaType.APPLICATION_JSON)
- @ApiOperation("List contents of a bucket")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response listBucket(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket,
- @DefaultValue(Header.OZONE_INFO_QUERY_KEY)
- @QueryParam(Header.OZONE_INFO_QUERY_TAG)
- String info,
- @QueryParam(Header.OZONE_LIST_QUERY_PREFIX)
- String prefix,
- @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
- @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS)
- int maxKeys,
- @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY)
- String prevKey,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
deleted file mode 100644
index f9255f2..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Keys.java
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.Consumes;
-import javax.ws.rs.DELETE;
-import javax.ws.rs.GET;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-import java.io.InputStream;
-
-/**
- * This interface defines operations permitted on a key.
- */
-@Path("/{volume}/{bucket}/{keys:.*}")
-@Api(tags = "key")
-public interface Keys {
-
- /**
- * Adds a key to an existing bucket. If the object already exists
- * this call will overwrite or add with new version number if the bucket
- * versioning is turned on.
- *
- * @param volume Storage Volume Name
- * @param bucket Name of the bucket
- * @param keys Name of the Object
- * @param is InputStream or File Data
- * @param req Request
- * @param headers http headers
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @PUT
- @Consumes(MediaType.WILDCARD)
- @ApiOperation(value = "Adds a key to an existing bucket.", notes = "If the "
- + "object already exists this call will overwrite or add with new version"
- + " number if the bucket versioning is turned on.")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response putKey(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket, @PathParam("keys") String keys,
- InputStream is, @Context Request req, @Context UriInfo info,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Gets the Key if it exists.
- *
- * @param volume Storage Volume
- * @param bucket Name of the bucket
- * @param keys Object Name
- * @param info Tag info
- * @param req Request
- * @param uriInfo Uri info
- * @param headers Http Header
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @GET
- @ApiOperation("Gets the Key if it exists.")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response getKey(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket, @PathParam("keys") String keys,
- @QueryParam(Header.OZONE_INFO_QUERY_TAG) String info,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Deletes an existing key.
- *
- * @param volume Storage Volume Name
- * @param bucket Name of the bucket
- * @param keys Name of the Object
- * @param req http Request
- * @param headers HttpHeaders
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @DELETE
- @ApiOperation("Deletes an existing key")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response deleteKey(@PathParam("volume") String volume,
- @PathParam("bucket") String bucket, @PathParam("keys") String keys,
- @Context Request req, @Context UriInfo info, @Context HttpHeaders headers)
- throws OzoneException;
-}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
deleted file mode 100644
index 6336c90..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/StorageHandler.java
+++ /dev/null
@@ -1,295 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * Storage handler Interface is the Interface between
- * REST protocol and file system.
- *
- * We will have two default implementations of this interface.
- * One for the local file system that is handy while testing
- * and another which will point to the HDFS backend.
- */
-@InterfaceAudience.Private
-public interface StorageHandler extends Closeable{
-
- /**
- * Creates a Storage Volume.
- *
- * @param args - Volume Name
- *
- * @throws IOException
- * @throws OzoneException
- */
- void createVolume(VolumeArgs args) throws IOException, OzoneException;
-
-
- /**
- * setVolumeOwner - sets the owner of the volume.
- *
- * @param args owner info is present in the args
- *
- * @throws IOException
- * @throws OzoneException
- */
- void setVolumeOwner(VolumeArgs args) throws IOException, OzoneException;
-
-
- /**
- * Set Volume Quota.
- *
- * @param args - Has Quota info
- * @param remove - true if the request is to remove the quota
- *
- * @throws IOException
- * @throws OzoneException
- */
- void setVolumeQuota(VolumeArgs args, boolean remove)
- throws IOException, OzoneException;
-
- /**
- * Checks if a Volume exists and the user with a role specified has access
- * to the Volume.
- *
- * @param volume - Volume Name whose access permissions needs to be checked
- * @param acl - requested acls which needs to be checked for access
- *
- * @return - Boolean - True if the user with a role can access the volume.
- * This is possible for owners of the volume and admin users
- *
- * @throws IOException
- * @throws OzoneException
- */
- boolean checkVolumeAccess(String volume, OzoneAcl acl)
- throws IOException, OzoneException;
-
-
- /**
- * Returns the List of Volumes owned by the specific user.
- *
- * @param args - ListArgs
- *
- * @return - List of Volumes
- *
- * @throws IOException
- * @throws OzoneException
- */
- ListVolumes listVolumes(ListArgs args) throws IOException, OzoneException;
-
- /**
- * Deletes an Empty Volume.
- *
- * @param args - Volume Args
- *
- * @throws IOException
- * @throws OzoneException
- */
- void deleteVolume(VolumeArgs args) throws IOException, OzoneException;
-
-
- /**
- * Returns Info about the specified Volume.
- *
- * @param args - Volume Args
- *
- * @return VolumeInfo
- *
- * @throws IOException
- * @throws OzoneException
- */
- VolumeInfo getVolumeInfo(VolumeArgs args) throws IOException, OzoneException;
-
- /**
- * Creates a Bucket in specified Volume.
- *
- * @param args BucketArgs- BucketName, UserName and Acls
- *
- * @throws IOException
- */
- void createBucket(BucketArgs args) throws IOException, OzoneException;
-
- /**
- * Adds or Removes ACLs from a Bucket.
- *
- * @param args - BucketArgs
- *
- * @throws IOException
- */
- void setBucketAcls(BucketArgs args) throws IOException, OzoneException;
-
- /**
- * Enables or disables Bucket Versioning.
- *
- * @param args - BucketArgs
- *
- * @throws IOException
- */
- void setBucketVersioning(BucketArgs args) throws IOException, OzoneException;
-
- /**
- * Sets the Storage Class of a Bucket.
- *
- * @param args - BucketArgs
- *
- * @throws IOException
- */
- void setBucketStorageClass(BucketArgs args)
- throws IOException, OzoneException;
-
- /**
- * Deletes a bucket if it is empty.
- *
- * @param args Bucket args structure
- *
- * @throws IOException
- */
- void deleteBucket(BucketArgs args) throws IOException, OzoneException;
-
- /**
- * true if the bucket exists and user has read access
- * to the bucket else throws Exception.
- *
- * @param args Bucket args structure
- *
- * @throws IOException
- */
- void checkBucketAccess(BucketArgs args) throws IOException, OzoneException;
-
-
- /**
- * Returns all Buckets of a specified Volume.
- *
- * @param listArgs -- List Args.
- *
- * @return ListAllBuckets
- *
- * @throws OzoneException
- */
- ListBuckets listBuckets(ListArgs listArgs) throws
- IOException, OzoneException;
-
-
- /**
- * Returns Bucket's Metadata as a String.
- *
- * @param args Bucket args structure
- *
- * @return Info about the bucket
- *
- * @throws IOException
- */
- BucketInfo getBucketInfo(BucketArgs args) throws IOException, OzoneException;
-
- /**
- * Writes a key in an existing bucket.
- *
- * @param args KeyArgs
- *
- * @return InputStream
- *
- * @throws OzoneException
- */
- OutputStream newKeyWriter(KeyArgs args)
- throws IOException, OzoneException;
-
-
- /**
- * Tells the file system that the object has been written out
- * completely and it can do any house keeping operation that needs
- * to be done.
- *
- * @param args Key Args
- *
- * @param stream
- * @throws IOException
- */
- void commitKey(KeyArgs args, OutputStream stream)
- throws IOException, OzoneException;
-
-
- /**
- * Reads a key from an existing bucket.
- *
- * @param args KeyArgs
- *
- * @return LengthInputStream
- *
- * @throws IOException
- */
- LengthInputStream newKeyReader(KeyArgs args)
- throws IOException, OzoneException;
-
-
- /**
- * Deletes an existing key.
- *
- * @param args KeyArgs
- *
- * @throws OzoneException
- */
- void deleteKey(KeyArgs args) throws IOException, OzoneException;
-
-
- /**
- * Returns a list of Key.
- *
- * @param args KeyArgs
- *
- * @return BucketList
- *
- * @throws IOException
- */
- ListKeys listKeys(ListArgs args) throws IOException, OzoneException;
-
- /**
- * Get information of the specified Key.
- *
- * @param args Key Args
- *
- * @return KeyInfo
- *
- * @throws IOException
- * @throws OzoneException
- */
- KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException;
-
- /**
- * Closes all the opened resources.
- */
- void close();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
deleted file mode 100644
index a1d2e7c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/UserAuth.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.UserArgs;
-
-/**
- * This interface is used by Ozone to determine user identity.
- *
- * Please see concrete implementations for more information
- */
-@InterfaceAudience.Private
-public interface UserAuth {
- /**
- * Returns the user name as a string from the URI and HTTP headers.
- *
- * @param userArgs - userArgs
- *
- * @return String - User name
- *
- * @throws OzoneException
- */
- String getUser(UserArgs userArgs) throws OzoneException;
-
- /**
- * Returns all the Groups that user is a member of.
- *
- * @param userArgs - userArgs
- *
- * @return Array of Groups
- *
- * @throws OzoneException
- */
- String[] getGroups(UserArgs userArgs) throws OzoneException;
-
- /**
- * Returns true if a user is a Admin.
- *
- * @param userArgs - userArgs
- *
- * @return true if Admin , false otherwise
- *
- * @throws OzoneException -- Allows the underlying system
- * to throw, that error will get propagated to clients
- */
- boolean isAdmin(UserArgs userArgs) throws OzoneException;
-
- /**
- * Returns true if the request is Anonymous.
- *
- * @param userArgs - userArgs
- *
- * @return true if the request is anonymous, false otherwise.
- *
- * @throws OzoneException - Will be propagated back to end user
- */
- boolean isAnonymous(UserArgs userArgs) throws OzoneException;
-
- /**
- * Returns true if the name is a recognizable user in the system.
- *
- * @param userName - User Name to check
- * @param userArgs - userArgs
- *
- * @return true if the username string is the name of a valid user.
- *
- * @throws OzoneException - Will be propagated back to end user
- */
- boolean isUser(String userName, UserArgs userArgs) throws OzoneException;
-
- /**
- * Returns the x-ozone-user or the user on behalf of, This is
- * used in Volume creation path.
- *
- * @param userArgs - userArgs
- *
- * @return a user name if it has x-ozone-user args in header.
- *
- * @throws OzoneException
- */
- String getOzoneUser(UserArgs userArgs) throws OzoneException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
deleted file mode 100644
index 85b2240..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/Volume.java
+++ /dev/null
@@ -1,182 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.interfaces;
-
-import io.swagger.annotations.Api;
-import io.swagger.annotations.ApiImplicitParam;
-import io.swagger.annotations.ApiImplicitParams;
-import io.swagger.annotations.ApiOperation;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.client.rest.headers.Header;
-
-import javax.ws.rs.DELETE;
-import javax.ws.rs.DefaultValue;
-import javax.ws.rs.GET;
-import javax.ws.rs.POST;
-import javax.ws.rs.PUT;
-import javax.ws.rs.Path;
-import javax.ws.rs.PathParam;
-import javax.ws.rs.QueryParam;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.HttpHeaders;
-import javax.ws.rs.core.Request;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.UriInfo;
-
-/**
- * Volume Interface acts as the HTTP entry point for
- * volume related functionality.
- */
-@InterfaceAudience.Private
-@Path("/{volume}")
-@Api(tags = "volume")
-public interface Volume {
-
- /**
- * Creates a Volume owned by the user.
- *
- * Params :
- * Quota - Specifies the Maximum usable size by the user
- * the valid parameters for quota are <int>(<BYTES| MB|GB|TB>) | remove.
- * For example 10GB or "remove".
- *
- * @param volume Volume Name, this has to be unique at Ozone Level
- * @param quota Quota for this Storage Volume - <int>(<MB|GB|TB>) | remove
- * @param req - Request Object - Request Object
- * @param uriInfo - Http UriInfo
- * @param headers Http Headers HttpHeaders
- *
- * @return Response
- *
- * @throws OzoneException
- */
-
- @POST
- @ApiOperation("Creates a Volume owned by the user")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response createVolume(@PathParam("volume") String volume,
- @DefaultValue(Header.OZONE_QUOTA_UNDEFINED)
- @QueryParam(Header.OZONE_QUOTA_QUERY_TAG) String quota,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Updates a Volume owned by the user.
- *
- * Params :
- * Owner - Specifies the name of the owner
- * Quota - Specifies the Maximum usable size by the user
- * the valid parameters for quota are <int>(<MB|GB|TB>) | remove.
- * For example 10GB or "remove".
- *
- * @param volume Volume Name, this has to be unique at Ozone Level
- * @param quota Quota for this Storage Volume - <int>(<MB|GB|TB>) | remove
- * @param req - Request Object - Request Object
- * @param headers Http Headers HttpHeaders
- *
- * @return Response
- *
- * @throws OzoneException
- */
- @PUT
- @ApiOperation("Updates a Volume owned by the user")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response updateVolume(@PathParam("volume") String volume,
- @DefaultValue(Header.OZONE_QUOTA_UNDEFINED)
- @QueryParam(Header.OZONE_QUOTA_QUERY_TAG) String quota,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Deletes a Volume if it is empty.
- *
- * @param volume Storage Volume Name
- *
- * @return Response Response
- *
- * @throws OzoneException
- */
- @DELETE
- @ApiOperation("Deletes a Volume if it is empty")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response deleteVolume(@PathParam("volume") String volume,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
- /**
- * Returns Volume info. This API can be invoked either
- * by admin or the owner
- *
- * @param volume - Storage Volume Name
- * @param req - Http Req
- * @param headers - Http headers
- *
- * @return - Response
- *
- * @throws OzoneException
- */
- @GET
- @ApiOperation(value = "Returns Volume info", notes = "This API can be "
- + "invoked either by admin or the owner")
- @ApiImplicitParams({
- @ApiImplicitParam(name = "x-ozone-version", example = "v1", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "x-ozone-user", example = "user", required =
- true, paramType = "header"),
- @ApiImplicitParam(name = "Date", example = "Date: Mon, 26 Jun 2017 "
- + "04:23:30 GMT", required = true, paramType = "header"),
- @ApiImplicitParam(name = "Authorization", example = "OZONE", required =
- true, paramType = "header")})
- Response getVolumeInfo(@PathParam("volume") String volume,
- @DefaultValue(Header.OZONE_INFO_QUERY_BUCKET)
- @QueryParam(Header.OZONE_INFO_QUERY_TAG) String info,
- @QueryParam(Header.OZONE_LIST_QUERY_PREFIX) String prefix,
- @DefaultValue(Header.OZONE_DEFAULT_LIST_SIZE)
- @QueryParam(Header.OZONE_LIST_QUERY_MAXKEYS) int keys,
- @QueryParam(Header.OZONE_LIST_QUERY_PREVKEY) String prevKey,
- @QueryParam(Header.OZONE_LIST_QUERY_ROOTSCAN) boolean rootScan,
- @Context Request req, @Context UriInfo uriInfo,
- @Context HttpHeaders headers) throws OzoneException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
deleted file mode 100644
index 940f179..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/interfaces/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.web.interfaces;
-
-/**
- This package contains ozone client side libraries.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
deleted file mode 100644
index 9747eac..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/localstorage/LocalStorageHandler.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.web.localstorage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.client.io.LengthInputStream;
-import org.apache.hadoop.ozone.client.rest.OzoneException;
-import org.apache.hadoop.ozone.web.handlers.BucketArgs;
-import org.apache.hadoop.ozone.web.handlers.KeyArgs;
-import org.apache.hadoop.ozone.web.handlers.ListArgs;
-import org.apache.hadoop.ozone.web.handlers.VolumeArgs;
-import org.apache.hadoop.ozone.web.interfaces.StorageHandler;
-import org.apache.hadoop.ozone.OzoneAcl;
-import org.apache.hadoop.ozone.web.request.OzoneQuota;
-import org.apache.hadoop.ozone.web.response.BucketInfo;
-import org.apache.hadoop.ozone.web.response.KeyInfo;
-import org.apache.hadoop.ozone.web.response.ListBuckets;
-import org.apache.hadoop.ozone.web.response.ListKeys;
-import org.apache.hadoop.ozone.web.response.ListVolumes;
-import org.apache.hadoop.ozone.web.response.VolumeInfo;
-
-import java.io.IOException;
-import java.io.OutputStream;
-
-/**
- * PLEASE NOTE : This file is a dummy backend for test purposes and prototyping
- * effort only. It does not handle any Object semantics correctly, neither does
- * it take care of security.
- */
-@InterfaceAudience.Private
-public class LocalStorageHandler implements StorageHandler {
- private final Configuration conf;
-
- /**
- * Constructs LocalStorageHandler.
- *
- * @param conf ozone conf.
- */
- public LocalStorageHandler(Configuration conf) {
- this.conf = conf;
- }
-
- /**
- * Creates Storage Volume.
- *
- * @param args - volumeArgs
- * @throws IOException
- */
- @Override
- public void createVolume(VolumeArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.createVolume(args);
-
- }
-
- /**
- * setVolumeOwner - sets the owner of the volume.
- *
- * @param args volumeArgs
- * @throws IOException
- */
- @Override
- public void setVolumeOwner(VolumeArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.OWNER);
- }
-
- /**
- * Set Volume Quota Info.
- *
- * @param args - volumeArgs
- * @param remove - true if the request is to remove the quota
- * @throws IOException
- */
- @Override
- public void setVolumeQuota(VolumeArgs args, boolean remove)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
-
- if (remove) {
- OzoneQuota quota = new OzoneQuota();
- args.setQuota(quota);
- }
- oz.setVolumeProperty(args, OzoneMetadataManager.VolumeProperty.QUOTA);
- }
-
- /**
- * Checks if a Volume exists and the user specified has access to the volume.
- *
- * @param volume - Volume Name
- * @param acl - Ozone acl which needs to be compared for access
- * @return - Boolean - True if the user can modify the volume. This is
- * possible for owners of the volume and admin users
- * @throws IOException
- */
- @Override
- public boolean checkVolumeAccess(String volume, OzoneAcl acl)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.checkVolumeAccess(volume, acl);
- }
-
- /**
- * Returns Info about the specified Volume.
- *
- * @param args - volumeArgs
- * @return VolumeInfo
- * @throws IOException
- */
- @Override
- public VolumeInfo getVolumeInfo(VolumeArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.getVolumeInfo(args);
- }
-
- /**
- * Deletes an Empty Volume.
- *
- * @param args - Volume Args
- * @throws IOException
- */
- @Override
- public void deleteVolume(VolumeArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.deleteVolume(args);
-
- }
-
- /**
- * Returns the List of Volumes owned by the specific user.
- *
- * @param args - ListArgs
- * @return - List of Volumes
- * @throws IOException
- */
- @Override
- public ListVolumes listVolumes(ListArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.listVolumes(args);
- }
-
- /**
- * true if the bucket exists and user has read access to the bucket else
- * throws Exception.
- *
- * @param args Bucket args structure
- * @throws IOException
- */
- @Override
- public void checkBucketAccess(BucketArgs args)
- throws IOException, OzoneException {
-
- }
-
- /**
- * Creates a Bucket in specified Volume.
- *
- * @param args BucketArgs- BucketName, UserName and Acls
- * @throws IOException
- */
- @Override
- public void createBucket(BucketArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.createBucket(args);
- }
-
- /**
- * Adds or Removes ACLs from a Bucket.
- *
- * @param args - BucketArgs
- * @throws IOException
- */
- @Override
- public void setBucketAcls(BucketArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.ACLS);
- }
-
- /**
- * Enables or disables Bucket Versioning.
- *
- * @param args - BucketArgs
- * @throws IOException
- */
- @Override
- public void setBucketVersioning(BucketArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.VERSIONING);
-
- }
-
- /**
- * Sets the Storage Class of a Bucket.
- *
- * @param args - BucketArgs
- * @throws IOException
- */
- @Override
- public void setBucketStorageClass(BucketArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.setBucketProperty(args, OzoneMetadataManager.BucketProperty.STORAGETYPE);
-
- }
-
- /**
- * Deletes a bucket if it is empty.
- *
- * @param args Bucket args structure
- * @throws IOException
- */
- @Override
- public void deleteBucket(BucketArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.deleteBucket(args);
- }
-
- /**
- * Returns all Buckets of a specified Volume.
- *
- * @param args --User Args
- * @return ListAllBuckets
- * @throws OzoneException
- */
- @Override
- public ListBuckets listBuckets(ListArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.listBuckets(args);
- }
-
- /**
- * Returns Bucket's Metadata as a String.
- *
- * @param args Bucket args structure
- * @return Info about the bucket
- * @throws IOException
- */
- @Override
- public BucketInfo getBucketInfo(BucketArgs args)
- throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.getBucketInfo(args);
- }
-
- /**
- * Writes a key in an existing bucket.
- *
- * @param args KeyArgs
- * @return InputStream
- * @throws OzoneException
- */
- @Override
- public OutputStream newKeyWriter(KeyArgs args) throws IOException,
- OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.createKey(args);
- }
-
- /**
- * Tells the file system that the object has been written out completely and
- * it can do any house keeping operation that needs to be done.
- *
- * @param args Key Args
- * @param stream
- * @throws IOException
- */
- @Override
- public void commitKey(KeyArgs args, OutputStream stream) throws
- IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.commitKey(args, stream);
-
- }
-
- /**
- * Reads a key from an existing bucket.
- *
- * @param args KeyArgs
- * @return LengthInputStream
- * @throws IOException
- */
- @Override
- public LengthInputStream newKeyReader(KeyArgs args) throws IOException,
- OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.newKeyReader(args);
- }
-
- /**
- * Deletes an existing key.
- *
- * @param args KeyArgs
- * @throws OzoneException
- */
- @Override
- public void deleteKey(KeyArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- oz.deleteKey(args);
- }
-
- /**
- * Returns a list of Key.
- *
- * @param args KeyArgs
- * @return BucketList
- * @throws IOException
- */
- @Override
- public ListKeys listKeys(ListArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz =
- OzoneMetadataManager.getOzoneMetadataManager(conf);
- return oz.listKeys(args);
-
- }
-
- /**
- * Get information of the specified Key.
- *
- * @param args Key Args
- *
- * @return KeyInfo
- *
- * @throws IOException
- * @throws OzoneException
- */
- @Override
- public KeyInfo getKeyInfo(KeyArgs args) throws IOException, OzoneException {
- OzoneMetadataManager oz = OzoneMetadataManager
- .getOzoneMetadataManager(conf);
- return oz.getKeyInfo(args);
- }
-
- @Override
- public void close() {
- //No resource to close, do nothing.
- }
-
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
index c7af6a1..1f37d21 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/dfs-dust.js
@@ -38,10 +38,7 @@
'fmt_percentage': function (v) {
return Math.round(v * 100) / 100 + '%';
},
- 'elapsed': function(v) {
- //elapsed sec from epoch sec
- return Date.now() - v * 1000;
- },
+
'fmt_time': function (v) {
var s = Math.floor(v / 1000), h = Math.floor(s / 3600);
s -= h * 3600;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
deleted file mode 100644
index b8a5c0f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css
+++ /dev/null
@@ -1,2 +0,0 @@
-.nvd3 .nv-axis line,.nvd3 .nv-axis path{fill:none;shape-rendering:crispEdges}.nv-brush .extent,.nvd3 .background path,.nvd3 .nv-axis line,.nvd3 .nv-axis path{shape-rendering:crispEdges}.nv-distx,.nv-disty,.nv-noninteractive,.nvd3 .nv-axis,.nvd3.nv-pie .nv-label,.nvd3.nv-sparklineplus g.nv-hoverValue{pointer-events:none}.nvd3 .nv-axis{opacity:1}.nvd3 .nv-axis.nv-disabled,.nvd3 .nv-controlsWrap .nv-legend .nv-check-box .nv-check{opacity:0}.nvd3 .nv-axis path{stroke:#000;stroke-opacity:.75}.nvd3 .nv-axis path.domain{stroke-opacity:.75}.nvd3 .nv-axis.nv-x path.domain{stroke-opacity:0}.nvd3 .nv-axis line{stroke:#e5e5e5}.nvd3 .nv-axis .zero line, .nvd3 .nv-axis line.zero{stroke-opacity:.75}.nvd3 .nv-axis .nv-axisMaxMin text{font-weight:700}.nvd3 .x .nv-axis .nv-axisMaxMin text,.nvd3 .x2 .nv-axis .nv-axisMaxMin text,.nvd3 .x3 .nv-axis .nv-axisMaxMin text{text-anchor:middle}.nvd3 .nv-bars rect{fill-opacity:.75;transition:fill-opacity 250ms linear}.nvd3 .nv-bars rect.hover{fill-opacity:1}.nv
d3 .nv-bars .hover rect{fill:#add8e6}.nvd3 .nv-bars text{fill:transparent}.nvd3 .nv-bars .hover text{fill:rgba(0,0,0,1)}.nvd3 .nv-discretebar .nv-groups rect,.nvd3 .nv-multibar .nv-groups rect,.nvd3 .nv-multibarHorizontal .nv-groups rect{stroke-opacity:0;transition:fill-opacity 250ms linear}.with-transitions .nv-candlestickBar .nv-ticks .nv-tick,.with-transitions .nvd3 .nv-groups .nv-point{transition:stroke-width 250ms linear,stroke-opacity 250ms linear}.nvd3 .nv-candlestickBar .nv-ticks rect:hover,.nvd3 .nv-discretebar .nv-groups rect:hover,.nvd3 .nv-multibar .nv-groups rect:hover,.nvd3 .nv-multibarHorizontal .nv-groups rect:hover{fill-opacity:1}.nvd3 .nv-discretebar .nv-groups text,.nvd3 .nv-multibarHorizontal .nv-groups text{font-weight:700;fill:rgba(0,0,0,1);stroke:transparent}.nvd3 .nv-boxplot circle{fill-opacity:.5}.nvd3 .nv-boxplot circle:hover,.nvd3 .nv-boxplot rect:hover{fill-opacity:1}.nvd3 line.nv-boxplot-median{stroke:#000}.nv-boxplot-tick:hover{stroke-width:2.5px}.nvd3.
nv-bullet{font:10px sans-serif}.nvd3.nv-bullet .nv-measure{fill-opacity:.8}.nvd3.nv-bullet .nv-measure:hover{fill-opacity:1}.nvd3.nv-bullet .nv-marker{stroke:#000;stroke-width:2px}.nvd3.nv-bullet .nv-markerTriangle{stroke:#000;fill:#fff;stroke-width:1.5px}.nvd3.nv-bullet .nv-markerLine{stroke:#000;stroke-width:1.5px}.nvd3.nv-bullet .nv-tick line{stroke:#666;stroke-width:.5px}.nvd3.nv-bullet .nv-range.nv-s0{fill:#eee}.nvd3.nv-bullet .nv-range.nv-s1{fill:#ddd}.nvd3.nv-bullet .nv-range.nv-s2{fill:#ccc}.nvd3.nv-bullet .nv-title{font-size:14px;font-weight:700}.nvd3.nv-bullet .nv-subtitle{fill:#999}.nvd3.nv-bullet .nv-range{fill:#bababa;fill-opacity:.4}.nvd3.nv-bullet .nv-range:hover{fill-opacity:.7}.nvd3.nv-candlestickBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.positive rect{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-candlestickBar .nv-ticks .nv-tick.negative rect{stroke:#d62728;fill:#d6272
8}.nvd3.nv-candlestickBar .nv-ticks line{stroke:#333}.nv-force-node{stroke:#fff;stroke-width:1.5px}.nv-force-link{stroke:#999;stroke-opacity:.6}.nv-force-node text{stroke-width:0}.nvd3 .nv-check-box .nv-box{fill-opacity:0;stroke-width:2}.nvd3 .nv-check-box .nv-check{fill-opacity:0;stroke-width:4}.nvd3 .nv-series.nv-disabled .nv-check-box .nv-check{fill-opacity:0;stroke-opacity:0}.nvd3.nv-linePlusBar .nv-bar rect{fill-opacity:.75}.nvd3.nv-linePlusBar .nv-bar rect:hover{fill-opacity:1}.nvd3 .nv-groups path.nv-line{fill:none}.nvd3 .nv-groups path.nv-area{stroke:none}.nvd3.nv-line .nvd3.nv-scatter .nv-groups .nv-point{fill-opacity:0;stroke-opacity:0}.nvd3.nv-scatter.nv-single-point .nv-groups .nv-point{fill-opacity:.5!important;stroke-opacity:.5!important}.nvd3 .nv-groups .nv-point.hover,.nvd3.nv-scatter .nv-groups .nv-point.hover{stroke-width:7px;fill-opacity:.95!important;stroke-opacity:.95!important}.nvd3 .nv-point-paths path{stroke:#aaa;stroke-opacity:0;fill:#eee;fill-opacity:0}.nvd
3 .nv-indexLine{cursor:ew-resize}svg.nvd3-svg{-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;display:block;width:100%;height:100%}.nvtooltip.with-3d-shadow,.with-3d-shadow .nvtooltip{box-shadow:0 5px 10px rgba(0,0,0,.2);border-radius:5px}.nvd3 text{font:400 12px Arial,sans-serif}.nvd3 .title{font:700 14px Arial,sans-serif}.nvd3 .nv-background{fill:#fff;fill-opacity:0}.nvd3.nv-noData{font-size:18px;font-weight:700}.nv-brush .extent{fill-opacity:.125}.nv-brush .resize path{fill:#eee;stroke:#666}.nvd3 .nv-legend .nv-series{cursor:pointer}.nvd3 .nv-legend .nv-disabled circle{fill-opacity:0}.nvd3 .nv-brush .extent{fill-opacity:0!important}.nvd3 .nv-brushBackground rect{stroke:#000;stroke-width:.4;fill:#fff;fill-opacity:.7}@media print{.nvd3 text{stroke-width:0;fill-opacity:1}}.nvd3.nv-ohlcBar .nv-ticks .nv-tick{stroke-width:1px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.hover{stroke-width:2px}.nvd3.nv-ohlcBar .nv-ticks .nv-tick.positive{stroke:#2ca02c}.nvd3
.nv-ohlcBar .nv-ticks .nv-tick.negative{stroke:#d62728}.nvd3 .background path{fill:none;stroke:#EEE;stroke-opacity:.4}.nvd3 .foreground path{fill:none;stroke-opacity:.7}.nvd3 .nv-parallelCoordinates-brush .extent{fill:#fff;fill-opacity:.6;stroke:gray;shape-rendering:crispEdges}.nvd3 .nv-parallelCoordinates .hover{fill-opacity:1;stroke-width:3px}.nvd3 .missingValuesline line{fill:none;stroke:#000;stroke-width:1;stroke-opacity:1;stroke-dasharray:5,5}.nvd3.nv-pie .nv-pie-title{font-size:24px;fill:rgba(19,196,249,.59)}.nvd3.nv-pie .nv-slice text{stroke:#000;stroke-width:0}.nvd3.nv-pie path{transition:fill-opacity 250ms linear,stroke-width 250ms linear,stroke-opacity 250ms linear;stroke:#fff;stroke-width:1px;stroke-opacity:1;fill-opacity:.7}.nvd3.nv-pie .hover path{fill-opacity:1}.nvd3.nv-pie .nv-label rect{fill-opacity:0;stroke-opacity:0}.nvd3 .nv-groups .nv-point.hover{stroke-width:20px;stroke-opacity:.5}.nvd3 .nv-scatter .nv-point.hover{fill-opacity:1}.nvd3.nv-sparkline path{fill:none
}.nvd3.nv-sparklineplus .nv-hoverValue line{stroke:#333;stroke-width:1.5px}.nvd3.nv-sparklineplus,.nvd3.nv-sparklineplus g{pointer-events:all}.nvd3 .nv-interactiveGuideLine,.nvtooltip{pointer-events:none}.nvd3 .nv-hoverArea{fill-opacity:0;stroke-opacity:0}.nvd3.nv-sparklineplus .nv-xValue,.nvd3.nv-sparklineplus .nv-yValue{stroke-width:0;font-size:.9em;font-weight:400}.nvd3.nv-sparklineplus .nv-yValue{stroke:#f66}.nvd3.nv-sparklineplus .nv-maxValue{stroke:#2ca02c;fill:#2ca02c}.nvd3.nv-sparklineplus .nv-minValue{stroke:#d62728;fill:#d62728}.nvd3.nv-sparklineplus .nv-currentValue{font-weight:700;font-size:1.1em}.nvtooltip h3,.nvtooltip table td.key{font-weight:400}.nvd3.nv-stackedarea path.nv-area{fill-opacity:.7;stroke-opacity:0;transition:fill-opacity 250ms linear,stroke-opacity 250ms linear}.nvd3.nv-stackedarea path.nv-area.hover{fill-opacity:.9}.nvd3.nv-stackedarea .nv-groups .nv-point{stroke-opacity:0;fill-opacity:0}.nvtooltip{position:absolute;color:rgba(0,0,0,1);padding:1px;z-in
dex:10000;display:block;font-family:Arial,sans-serif;font-size:13px;text-align:left;white-space:nowrap;-webkit-user-select:none;-moz-user-select:none;-ms-user-select:none;user-select:none;background:rgba(255,255,255,.8);border:1px solid rgba(0,0,0,.5);border-radius:4px}.nvtooltip h3,.nvtooltip p{margin:0;text-align:center}.nvtooltip.with-transitions,.with-transitions .nvtooltip{transition:opacity 50ms linear;transition-delay:200ms}.nvtooltip.x-nvtooltip,.nvtooltip.y-nvtooltip{padding:8px}.nvtooltip h3{padding:4px 14px;line-height:18px;background-color:rgba(247,247,247,.75);color:rgba(0,0,0,1);border-bottom:1px solid #ebebeb;border-radius:5px 5px 0 0}.nvtooltip p{padding:5px 14px}.nvtooltip span{display:inline-block;margin:2px 0}.nvtooltip table{margin:6px;border-spacing:0}.nvtooltip table td{padding:2px 9px 2px 0;vertical-align:middle}.nvtooltip table td.key.total{font-weight:700}.nvtooltip table td.value{text-align:right;font-weight:700}.nvtooltip table td.percent{color:#a9a9a9}.nv
tooltip table tr.highlight td{padding:1px 9px 1px 0;border-bottom-style:solid;border-bottom-width:1px;border-top-style:solid;border-top-width:1px}.nvtooltip table td.legend-color-guide div{vertical-align:middle;width:12px;height:12px;border:1px solid #999}.nvtooltip .footer{padding:3px;text-align:center}.nvtooltip-pending-removal{pointer-events:none;display:none}.nvd3 line.nv-guideline{stroke:#ccc}
-/*# sourceMappingURL=nv.d3.min.css.map */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
deleted file mode 100644
index 63380e6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/static/nvd3-1.8.5.min.css.map
+++ /dev/null
@@ -1 +0,0 @@
-{"version":3,"sources":["build/nv.d3.css"],"names":[],"mappings":"AAqBA,oBAfA,oBAgBI,KAAM,KAiWN,gBAAiB,WA/ErB,kBA+DA,uBAlVA,oBAfA,oBAiXI,gBAAiB,WAqErB,UAAW,UAJX,mBAvbA,eAoaA,uBAgCA,uCACI,eAAgB,KArcpB,eAEI,QAAS,EAuCb,2BAsJA,0DACI,QAAS,EA3Lb,oBAEI,OAAQ,KACR,eAAgB,IAIpB,2BACI,eAAgB,IAGpB,gCACI,eAAgB,EAGpB,oBAEI,OAAQ,QAIZ,0BACI,0BACA,eAAgB,IAGpB,mCACI,YAAa,IAGjB,sCACA,uCACA,uCACI,YAAa,OAOjB,oBACI,aAAc,IAEd,WAAY,aAAa,MAAM,OAGnC,0BACI,aAAc,EAGlB,2BACI,KAAM,QAGV,oBACI,KAAM,YAGV,2BACI,KAAM,cAKV,sCAFA,mCACA,6CAEI,eAAgB,EAEhB,WAAY,aAAa,MAAM,OA8EnC,wDAwEA,6CACI,WAAY,aAAa,MAAM,OAAQ,eAAe,MAAM,OAlJhE,8CACA,4CAHA,yCACA,mDAGI,aAAc,EAGlB,sCACA,6CACI,YAAa,IACb,KAAM,cACN,OAAQ,YAIZ,yBACE,aAAc,GAGhB,+BAIA,6BAHE,aAAc,EAOhB,6BACE,OAAQ,KAGV,uBACE,aAAc,MAGhB,gBAAkB,KAAM,KAAK,WAC7B,4BAA8B,aAAc,GAC5C,kCAAoC,aAAc,EAClD,2BAA6B,OAAQ,KAAM,aAAc,IACzD,mCAAqC,OAAQ,KAAM,KAAM,KAAM,aAAc,MAC7E,+BAAiC,OAAQ,KAAM,aAAc,MAC7D,8BAAgC,OAAQ,KAAM,aAAc,KAC5D,gCAAkC,KAAM,KACxC,gCAAkC,KAAM,KACxC,gCAAkC,KAAM,KACxC,0BAA4B,UAAW,KAAM,Y
AAa,IAC1D,6BAA+B,KAAM,KAErC,0BACI,KAAM,QACN,aAAc,GAGlB,gCACI,aAAc,GAGlB,2CACI,aAAc,IAGlB,iDACI,aAAc,IAGlB,yDACI,OAAQ,QACR,KAAM,QAGV,yDACI,OAAQ,QACR,KAAM,QAOV,uCACI,OAAQ,KAGZ,eACI,OAAQ,KACR,aAAc,MAGlB,eACI,OAAQ,KACR,eAAgB,GAGpB,oBACI,aAAc,EAOlB,4BACI,aAAa,EACb,aAAa,EAGjB,8BACI,aAAa,EACb,aAAa,EAGjB,qDACI,aAAa,EACb,eAAe,EAQnB,kCACI,aAAc,IAGlB,wCACI,aAAc,EAElB,8BACI,KAAM,KAGV,8BACI,OAAQ,KAGZ,oDACI,aAAc,EACd,eAAgB,EAGpB,sDACI,aAAc,aACd,eAAgB,aASpB,iCADA,4CAEI,aAAc,IACd,aAAc,cACd,eAAgB,cAIpB,2BACI,OAAQ,KACR,eAAgB,EAChB,KAAM,KACN,aAAc,EAIlB,oBACI,OAAQ,UAUZ,aACI,oBAAqB,KAClB,iBAAkB,KACjB,gBAAiB,KACb,YAAa,KACrB,QAAS,MACT,MAAM,KACN,OAAO,KAMX,0BAA2B,2BACvB,WAAY,EAAE,IAAI,KAAK,eACvB,cAAe,IAInB,WACI,KAAM,IAAO,KAAK,MAAO,WAG7B,aACI,KAAM,IAAK,KAAK,MAAO,WAG3B,qBACI,KAAM,KACN,aAAc,EAGlB,gBACI,UAAW,KACX,YAAa,IAQjB,kBACI,aAAc,KAIlB,uBACI,KAAM,KACN,OAAQ,KAQZ,4BACI,OAAQ,QAGZ,qCACI,aAAc,EAIlB,wBACI,aAAc,YAGlB,+BACI,OAAQ,KACR,aAAc,GACd,KAAM,KACN,aAAc,GAOlB,aACI,WACI,aAAc,EACd,aAAc,GAItB,oCACI,aAAc,IAGlB,0C
ACI,aAAc,IAGlB,6CACI,OAAQ,QAGZ,6CACI,OAAQ,QAIZ,uBACI,KAAM,KACN,OAAQ,KACR,eAAgB,GAIpB,uBACI,KAAM,KACN,eAAgB,GAGpB,4CACI,KAAM,KACN,aAAc,GACd,OAAQ,KACR,gBAAiB,WAGrB,qCACI,aAAc,EACjB,aAAc,IAIf,8BACE,KAAM,KACN,OAAQ,KACR,aAAc,EACd,eAAgB,EAChB,iBAAkB,EAAG,EAQvB,2BACI,UAAW,KACX,KAAM,qBAGV,4BACI,OAAQ,KACR,aAAc,EAGlB,kBAbI,WAAY,aAAa,MAAM,OAAQ,aAAa,MAAM,OAAQ,eAAe,MAAM,OAcvF,OAAQ,KACR,aAAc,IACd,eAAgB,EAIhB,aAAc,GAGlB,yBACI,aAAc,EAOlB,4BACI,aAAc,EACd,eAAgB,EAIpB,iCACI,aAAc,KACd,eAAgB,GAGpB,kCACI,aAAc,EAYlB,wBACI,KAAM,KAOV,2CACI,OAAQ,KACR,aAAc,MAGlB,uBACA,yBACI,eAAgB,IAsLpB,+BApIA,WAqII,eAAe,KApLnB,oBACI,aAAc,EACd,eAAgB,EAGpB,kCACA,kCACI,aAAc,EACd,UAAW,KACX,YAAa,IAGjB,kCACI,OAAQ,KAGZ,oCACI,OAAQ,QACR,KAAM,QAGV,oCACI,OAAQ,QACR,KAAM,QAGV,wCACI,YAAa,IACb,UAAW,MAgEf,cAoCA,wBACI,YAAa,IAlGjB,kCACI,aAAc,GACd,eAAgB,EAChB,WAAY,aAAa,MAAM,OAAQ,eAAe,MAAM,OAGhE,wCACI,aAAc,GAIlB,0CACI,eAAgB,EAChB,aAAc,EAGlB,WACI,SAAU,SAEV,MAAO,cACP,QAAS,IAET,QAAS,MACT,QAAS,MAET,YAAa,MAAO,WACpB,UAAW,KACX,WAAY,KAGZ,YAAa,OAEb,oBAA
qB,KAElB,iBAAkB,KAEjB,gBAAiB,KAEb,YAAa,KAIrB,WAAY,qBACZ,OAAQ,IAAI,MAAM,eAClB,cAAe,IAiBnB,cAcA,aACI,OAAQ,EAER,WAAY,OA5BhB,4BAA6B,6BACzB,WAAY,QAAQ,KAAK,OAEzB,iBAAkB,MAGtB,uBACA,uBACI,QAAS,IAGb,cAEI,QAAS,IAAI,KACb,YAAa,KAEb,iBAAkB,sBAClB,MAAO,cAGP,cAAe,IAAI,MAAM,QAEzB,cAAe,IAAI,IAAI,EAAE,EAG7B,aAEI,QAAS,IAAI,KAIjB,gBACI,QAAS,aACT,OAAQ,IAAI,EAGhB,iBACI,OAAQ,IACR,eAAe,EAInB,oBACI,QAAS,IAAI,IAAI,IAAI,EACrB,eAAgB,OAOpB,8BACI,YAAa,IAGjB,0BACI,WAAY,MACZ,YAAa,IAGjB,4BACI,MAAO,QAGX,iCACI,QAAS,IAAI,IAAI,IAAI,EACrB,oBAAqB,MACrB,oBAAqB,IACrB,iBAAkB,MAClB,iBAAkB,IAGtB,2CAGI,eAAgB,OAIhB,MAAO,KACP,OAAQ,KACR,OAAQ,IAAI,MAAM,KAGtB,mBACI,QAAS,IACT,WAAY,OAGhB,2BACI,eAAgB,KAChB,QAAS,KAWb,wBACI,OAAQ"}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/51] [partial] hadoop git commit: HDFS-13258. Ozone: restructure
Hdsl/Ozone code to separated maven subprojects. Contributed by Elek Marton,
Mukul Kumar Singh, Xiaoyu Yao, Ajay Kumar, Anu Engineer, Lokesh Jain,
Nanda Kumar.
Posted by ae...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
deleted file mode 100644
index 95bf107..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/RegisterEndpointTask.java
+++ /dev/null
@@ -1,198 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.container.common.statemachine
- .EndpointStateMachine;
-
-import org.apache.hadoop.ozone.protocol.proto
- .StorageContainerDatanodeProtocolProtos.ContainerNodeIDProto;
-import org.apache.hadoop.scm.ScmConfigKeys;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Future;
-
-/**
- * Register a container with SCM.
- */
-public final class RegisterEndpointTask implements
- Callable<EndpointStateMachine.EndPointStates> {
- static final Logger LOG = LoggerFactory.getLogger(RegisterEndpointTask.class);
-
- private final EndpointStateMachine rpcEndPoint;
- private final Configuration conf;
- private Future<EndpointStateMachine.EndPointStates> result;
- private ContainerNodeIDProto containerNodeIDProto;
-
- /**
- * Creates a register endpoint task.
- *
- * @param rpcEndPoint - endpoint
- * @param conf - conf
- */
- @VisibleForTesting
- public RegisterEndpointTask(EndpointStateMachine rpcEndPoint,
- Configuration conf) {
- this.rpcEndPoint = rpcEndPoint;
- this.conf = conf;
-
- }
-
- /**
- * Get the ContainerNodeID Proto.
- *
- * @return ContainerNodeIDProto
- */
- public ContainerNodeIDProto getContainerNodeIDProto() {
- return containerNodeIDProto;
- }
-
- /**
- * Set the contiainerNodeID Proto.
- *
- * @param containerNodeIDProto - Container Node ID.
- */
- public void setContainerNodeIDProto(ContainerNodeIDProto
- containerNodeIDProto) {
- this.containerNodeIDProto = containerNodeIDProto;
- }
-
- /**
- * Computes a result, or throws an exception if unable to do so.
- *
- * @return computed result
- * @throws Exception if unable to compute a result
- */
- @Override
- public EndpointStateMachine.EndPointStates call() throws Exception {
-
- if (getContainerNodeIDProto() == null) {
- LOG.error("Container ID proto cannot be null in RegisterEndpoint task, " +
- "shutting down the endpoint.");
- return rpcEndPoint.setState(EndpointStateMachine.EndPointStates.SHUTDOWN);
- }
-
- rpcEndPoint.lock();
- try {
- DatanodeID dnNodeID = DatanodeID.getFromProtoBuf(
- getContainerNodeIDProto().getDatanodeID());
-
- // TODO : Add responses to the command Queue.
- rpcEndPoint.getEndPoint().register(dnNodeID,
- conf.getStrings(ScmConfigKeys.OZONE_SCM_NAMES));
- EndpointStateMachine.EndPointStates nextState =
- rpcEndPoint.getState().getNextState();
- rpcEndPoint.setState(nextState);
- rpcEndPoint.zeroMissedCount();
- } catch (IOException ex) {
- rpcEndPoint.logIfNeeded(ex
- );
- } finally {
- rpcEndPoint.unlock();
- }
-
- return rpcEndPoint.getState();
- }
-
- /**
- * Returns a builder class for RegisterEndPoint task.
- *
- * @return Builder.
- */
- public static Builder newBuilder() {
- return new Builder();
- }
-
- /**
- * Builder class for RegisterEndPoint task.
- */
- public static class Builder {
- private EndpointStateMachine endPointStateMachine;
- private Configuration conf;
- private ContainerNodeIDProto containerNodeIDProto;
-
- /**
- * Constructs the builder class.
- */
- public Builder() {
- }
-
- /**
- * Sets the endpoint state machine.
- *
- * @param rpcEndPoint - Endpoint state machine.
- * @return Builder
- */
- public Builder setEndpointStateMachine(EndpointStateMachine rpcEndPoint) {
- this.endPointStateMachine = rpcEndPoint;
- return this;
- }
-
- /**
- * Sets the Config.
- *
- * @param config - config
- * @return Builder.
- */
- public Builder setConfig(Configuration config) {
- this.conf = config;
- return this;
- }
-
- /**
- * Sets the NodeID.
- *
- * @param nodeID - NodeID proto
- * @return Builder
- */
- public Builder setNodeID(ContainerNodeIDProto nodeID) {
- this.containerNodeIDProto = nodeID;
- return this;
- }
-
- public RegisterEndpointTask build() {
- if (endPointStateMachine == null) {
- LOG.error("No endpoint specified.");
- throw new IllegalArgumentException("A valid endpoint state machine is" +
- " needed to construct RegisterEndPoint task");
- }
-
- if (conf == null) {
- LOG.error("No config specified.");
- throw new IllegalArgumentException("A valid configration is needed to" +
- " construct RegisterEndpoint task");
- }
-
- if (containerNodeIDProto == null) {
- LOG.error("No nodeID specified.");
- throw new IllegalArgumentException("A vaild Node ID is needed to " +
- "construct RegisterEndpoint task");
- }
-
- RegisterEndpointTask task = new RegisterEndpointTask(this
- .endPointStateMachine, this.conf);
- task.setContainerNodeIDProto(containerNodeIDProto);
- return task;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
deleted file mode 100644
index fa59234..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/VersionEndpointTask.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.container.common.statemachine.EndpointStateMachine;
-import org.apache.hadoop.ozone.protocol.VersionResponse;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMVersionResponseProto;
-
-import java.io.IOException;
-import java.util.concurrent.Callable;
-
-/**
- * Task that returns version.
- */
-public class VersionEndpointTask implements
- Callable<EndpointStateMachine.EndPointStates> {
- private final EndpointStateMachine rpcEndPoint;
- private final Configuration configuration;
-
- public VersionEndpointTask(EndpointStateMachine rpcEndPoint,
- Configuration conf) {
- this.rpcEndPoint = rpcEndPoint;
- this.configuration = conf;
- }
-
- /**
- * Computes a result, or throws an exception if unable to do so.
- *
- * @return computed result
- * @throws Exception if unable to compute a result
- */
- @Override
- public EndpointStateMachine.EndPointStates call() throws Exception {
- rpcEndPoint.lock();
- try{
- SCMVersionResponseProto versionResponse =
- rpcEndPoint.getEndPoint().getVersion(null);
- rpcEndPoint.setVersion(VersionResponse.getFromProtobuf(versionResponse));
-
- EndpointStateMachine.EndPointStates nextState =
- rpcEndPoint.getState().getNextState();
- rpcEndPoint.setState(nextState);
- rpcEndPoint.zeroMissedCount();
- } catch (IOException ex) {
- rpcEndPoint.logIfNeeded(ex);
- } finally {
- rpcEndPoint.unlock();
- }
- return rpcEndPoint.getState();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
deleted file mode 100644
index 1122598..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/endpoint/package-info.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.container.common.states.endpoint;
-/**
- This package contains code for RPC endpoints transitions.
- */
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
deleted file mode 100644
index 92c953f..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/states/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.states;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
deleted file mode 100644
index 3a5ce18..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServer.java
+++ /dev/null
@@ -1,128 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import com.google.common.base.Preconditions;
-import io.netty.bootstrap.ServerBootstrap;
-import io.netty.channel.Channel;
-import io.netty.channel.EventLoopGroup;
-import io.netty.channel.nio.NioEventLoopGroup;
-import io.netty.channel.socket.nio.NioServerSocketChannel;
-import io.netty.handler.logging.LogLevel;
-import io.netty.handler.logging.LoggingHandler;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-
-/**
- * Creates a netty server endpoint that acts as the communication layer for
- * Ozone containers.
- */
-public final class XceiverServer implements XceiverServerSpi {
- private static final Logger
- LOG = LoggerFactory.getLogger(XceiverServer.class);
- private int port;
- private final ContainerDispatcher storageContainer;
-
- private EventLoopGroup bossGroup;
- private EventLoopGroup workerGroup;
- private Channel channel;
-
- /**
- * Constructs a netty server class.
- *
- * @param conf - Configuration
- */
- public XceiverServer(Configuration conf,
- ContainerDispatcher dispatcher) {
- Preconditions.checkNotNull(conf);
-
- this.port = conf.getInt(OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
- // Get an available port on current node and
- // use that as the container port
- if (conf.getBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT,
- OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT_DEFAULT)) {
- try (ServerSocket socket = new ServerSocket()) {
- socket.setReuseAddress(true);
- SocketAddress address = new InetSocketAddress(0);
- socket.bind(address);
- this.port = socket.getLocalPort();
- LOG.info("Found a free port for the server : {}", this.port);
- } catch (IOException e) {
- LOG.error("Unable find a random free port for the server, "
- + "fallback to use default port {}", this.port, e);
- }
- }
- this.storageContainer = dispatcher;
- }
-
- @Override
- public int getIPCPort() {
- return this.port;
- }
-
- /**
- * Returns the Replication type supported by this end-point.
- *
- * @return enum -- {Stand_Alone, Ratis, Chained}
- */
- @Override
- public OzoneProtos.ReplicationType getServerType() {
- return OzoneProtos.ReplicationType.STAND_ALONE;
- }
-
- @Override
- public void start() throws IOException {
- bossGroup = new NioEventLoopGroup();
- workerGroup = new NioEventLoopGroup();
- channel = new ServerBootstrap()
- .group(bossGroup, workerGroup)
- .channel(NioServerSocketChannel.class)
- .handler(new LoggingHandler(LogLevel.INFO))
- .childHandler(new XceiverServerInitializer(storageContainer))
- .bind(port)
- .syncUninterruptibly()
- .channel();
- }
-
- @Override
- public void stop() {
- if (storageContainer != null) {
- storageContainer.shutdown();
- }
- if (bossGroup != null) {
- bossGroup.shutdownGracefully();
- }
- if (workerGroup != null) {
- workerGroup.shutdownGracefully();
- }
- if (channel != null) {
- channel.close().awaitUninterruptibly();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
deleted file mode 100644
index c4a8f53..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerHandler.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import io.netty.channel.ChannelHandlerContext;
-import io.netty.channel.SimpleChannelInboundHandler;
-import org.slf4j.LoggerFactory;
-import org.slf4j.Logger;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-
-/**
- * Netty server handlers that respond to Network events.
- */
-public class XceiverServerHandler extends
- SimpleChannelInboundHandler<ContainerCommandRequestProto> {
-
- static final Logger LOG = LoggerFactory.getLogger(XceiverServerHandler.class);
- private final ContainerDispatcher dispatcher;
-
- /**
- * Constructor for server handler.
- * @param dispatcher - Dispatcher interface
- */
- public XceiverServerHandler(ContainerDispatcher dispatcher) {
- this.dispatcher = dispatcher;
- }
-
- /**
- * <strong>Please keep in mind that this method will be renamed to {@code
- * messageReceived(ChannelHandlerContext, I)} in 5.0.</strong>
- * <p>
- * Is called for each message of type {@link ContainerCommandRequestProto}.
- *
- * @param ctx the {@link ChannelHandlerContext} which this {@link
- * SimpleChannelInboundHandler} belongs to
- * @param msg the message to handle
- * @throws Exception is thrown if an error occurred
- */
- @Override
- public void channelRead0(ChannelHandlerContext ctx,
- ContainerCommandRequestProto msg) throws
- Exception {
- ContainerCommandResponseProto response = this.dispatcher.dispatch(msg);
- LOG.debug("Writing the reponse back to client.");
- ctx.writeAndFlush(response);
-
- }
-
- /**
- * Calls {@link ChannelHandlerContext#fireExceptionCaught(Throwable)}
- * Sub-classes may override this method to change behavior.
- *
- * @param ctx - Channel Handler Context
- * @param cause - Exception
- */
- @Override
- public void exceptionCaught(ChannelHandlerContext ctx, Throwable cause)
- throws Exception {
- LOG.error("An exception caught in the pipeline : " + cause.toString());
- super.exceptionCaught(ctx, cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
deleted file mode 100644
index 4d32d86..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerInitializer.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import com.google.common.base.Preconditions;
-import io.netty.channel.ChannelInitializer;
-import io.netty.channel.ChannelPipeline;
-import io.netty.channel.socket.SocketChannel;
-import io.netty.handler.codec.protobuf.ProtobufDecoder;
-import io.netty.handler.codec.protobuf.ProtobufEncoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32FrameDecoder;
-import io.netty.handler.codec.protobuf.ProtobufVarint32LengthFieldPrepender;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-
-/**
- * Creates a channel for the XceiverServer.
- */
-public class XceiverServerInitializer extends ChannelInitializer<SocketChannel>{
- private final ContainerDispatcher dispatcher;
- public XceiverServerInitializer(ContainerDispatcher dispatcher) {
- Preconditions.checkNotNull(dispatcher);
- this.dispatcher = dispatcher;
- }
-
- /**
- * This method will be called once the Channel is registered. After
- * the method returns this instance will be removed from the {@link
- * ChannelPipeline}
- *
- * @param ch the which was registered.
- * @throws Exception is thrown if an error occurs. In that case the channel
- * will be closed.
- */
- @Override
- protected void initChannel(SocketChannel ch) throws Exception {
- ChannelPipeline pipeline = ch.pipeline();
- pipeline.addLast(new ProtobufVarint32FrameDecoder());
- pipeline.addLast(new ProtobufDecoder(ContainerCommandRequestProto
- .getDefaultInstance()));
- pipeline.addLast(new ProtobufVarint32LengthFieldPrepender());
- pipeline.addLast(new ProtobufEncoder());
- pipeline.addLast(new XceiverServerHandler(dispatcher));
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
deleted file mode 100644
index b61d7fd..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/XceiverServerSpi.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-
-import java.io.IOException;
-
-/** A server endpoint that acts as the communication layer for Ozone
- * containers. */
-public interface XceiverServerSpi {
- /** Starts the server. */
- void start() throws IOException;
-
- /** Stops a running server. */
- void stop();
-
- /** Get server IPC port. */
- int getIPCPort();
-
- /**
- * Returns the Replication type supported by this end-point.
- * @return enum -- {Stand_Alone, Ratis, Chained}
- */
- OzoneProtos.ReplicationType getServerType();
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
deleted file mode 100644
index 59c96f1..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/package-info.java
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server;
-
-/**
- * This package contains classes for the server of the storage container
- * protocol.
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
deleted file mode 100644
index 569fb23..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/ContainerStateMachine.java
+++ /dev/null
@@ -1,290 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandRequestProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.ContainerCommandResponseProto;
-import org.apache.hadoop.hdfs.ozone.protocol.proto.ContainerProtos.WriteChunkRequestProto;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.protocol.Message;
-import org.apache.ratis.protocol.RaftClientRequest;
-import org.apache.ratis.protocol.RaftPeerId;
-import org.apache.ratis.server.storage.RaftStorage;
-import org.apache.ratis.shaded.com.google.protobuf.ShadedProtoUtil;
-import org.apache.ratis.shaded.proto.RaftProtos.LogEntryProto;
-import org.apache.ratis.shaded.proto.RaftProtos.SMLogEntryProto;
-import org.apache.ratis.statemachine.impl.BaseStateMachine;
-import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
-import org.apache.ratis.statemachine.StateMachineStorage;
-import org.apache.ratis.statemachine.TransactionContext;
-import org.apache.ratis.statemachine.impl.TransactionContextImpl;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.apache.ratis.shaded.com.google.protobuf.ByteString;
-import com.google.protobuf.InvalidProtocolBufferException;
-
-import java.io.IOException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.ConcurrentHashMap;
-
-/** A {@link org.apache.ratis.statemachine.StateMachine} for containers.
- *
- * The stateMachine is responsible for handling different types of container
- * requests. The container requests can be divided into readonly and write
- * requests.
- *
- * Read only requests are classified in
- * {@link org.apache.hadoop.scm.XceiverClientRatis#isReadOnly}
- * and these readonly requests are replied from the {@link #query(Message)}.
- *
- * The write requests can be divided into requests with user data
- * (WriteChunkRequest) and other request without user data.
- *
- * Inorder to optimize the write throughput, the writeChunk request is
- * processed in 2 phases. The 2 phases are divided in
- * {@link #startTransaction(RaftClientRequest)}, in the first phase the user
- * data is written directly into the state machine via
- * {@link #writeStateMachineData} and in the second phase the
- * transaction is committed via {@link #applyTransaction(TransactionContext)}
- *
- * For the requests with no stateMachine data, the transaction is directly
- * committed through
- * {@link #applyTransaction(TransactionContext)}
- *
- * There are 2 ordering operation which are enforced right now in the code,
- * 1) Write chunk operation are executed after the create container operation,
- * the write chunk operation will fail otherwise as the container still hasn't
- * been created. Hence the create container operation has been split in the
- * {@link #startTransaction(RaftClientRequest)}, this will help in synchronizing
- * the calls in {@link #writeStateMachineData}
- *
- * 2) Write chunk commit operation is executed after write chunk state machine
- * operation. This will ensure that commit operation is sync'd with the state
- * machine operation.
- * */
-public class ContainerStateMachine extends BaseStateMachine {
- static final Logger LOG = LoggerFactory.getLogger(
- ContainerStateMachine.class);
- private final SimpleStateMachineStorage storage
- = new SimpleStateMachineStorage();
- private final ContainerDispatcher dispatcher;
- private ThreadPoolExecutor writeChunkExecutor;
- private final ConcurrentHashMap<Long, CompletableFuture<Message>>
- writeChunkFutureMap;
- private final ConcurrentHashMap<String, CompletableFuture<Message>>
- createContainerFutureMap;
-
- ContainerStateMachine(ContainerDispatcher dispatcher,
- ThreadPoolExecutor writeChunkExecutor) {
- this.dispatcher = dispatcher;
- this.writeChunkExecutor = writeChunkExecutor;
- this.writeChunkFutureMap = new ConcurrentHashMap<>();
- this.createContainerFutureMap = new ConcurrentHashMap<>();
- }
-
- @Override
- public StateMachineStorage getStateMachineStorage() {
- return storage;
- }
-
- @Override
- public void initialize(
- RaftPeerId id, RaftProperties properties, RaftStorage raftStorage)
- throws IOException {
- super.initialize(id, properties, raftStorage);
- storage.init(raftStorage);
- // TODO handle snapshots
-
- // TODO: Add a flag that tells you that initialize has been called.
- // Check with Ratis if this feature is done in Ratis.
- }
-
- @Override
- public TransactionContext startTransaction(RaftClientRequest request)
- throws IOException {
- final ContainerCommandRequestProto proto =
- getRequestProto(request.getMessage().getContent());
-
- final SMLogEntryProto log;
- if (proto.getCmdType() == ContainerProtos.Type.WriteChunk) {
- final WriteChunkRequestProto write = proto.getWriteChunk();
- // create the state machine data proto
- final WriteChunkRequestProto dataWriteChunkProto =
- WriteChunkRequestProto
- .newBuilder(write)
- .setStage(ContainerProtos.Stage.WRITE_DATA)
- .build();
- ContainerCommandRequestProto dataContainerCommandProto =
- ContainerCommandRequestProto
- .newBuilder(proto)
- .setWriteChunk(dataWriteChunkProto)
- .build();
-
- // create the log entry proto
- final WriteChunkRequestProto commitWriteChunkProto =
- WriteChunkRequestProto.newBuilder()
- .setPipeline(write.getPipeline())
- .setKeyName(write.getKeyName())
- .setChunkData(write.getChunkData())
- // skipping the data field as it is
- // already set in statemachine data proto
- .setStage(ContainerProtos.Stage.COMMIT_DATA)
- .build();
- ContainerCommandRequestProto commitContainerCommandProto =
- ContainerCommandRequestProto
- .newBuilder(proto)
- .setWriteChunk(commitWriteChunkProto)
- .build();
-
- log = SMLogEntryProto.newBuilder()
- .setData(getShadedByteString(commitContainerCommandProto))
- .setStateMachineData(getShadedByteString(dataContainerCommandProto))
- .build();
- } else if (proto.getCmdType() == ContainerProtos.Type.CreateContainer) {
- log = SMLogEntryProto.newBuilder()
- .setData(request.getMessage().getContent())
- .setStateMachineData(request.getMessage().getContent())
- .build();
- } else {
- log = SMLogEntryProto.newBuilder()
- .setData(request.getMessage().getContent())
- .build();
- }
- return new TransactionContextImpl(this, request, log);
- }
-
- private ByteString getShadedByteString(ContainerCommandRequestProto proto) {
- return ShadedProtoUtil.asShadedByteString(proto.toByteArray());
- }
-
- private ContainerCommandRequestProto getRequestProto(ByteString request)
- throws InvalidProtocolBufferException {
- return ContainerCommandRequestProto.parseFrom(
- ShadedProtoUtil.asByteString(request));
- }
-
- private Message runCommand(ContainerCommandRequestProto requestProto) {
- LOG.trace("dispatch {}", requestProto);
- ContainerCommandResponseProto response = dispatcher.dispatch(requestProto);
- LOG.trace("response {}", response);
- return () -> ShadedProtoUtil.asShadedByteString(response.toByteArray());
- }
-
- private CompletableFuture<Message> handleWriteChunk(
- ContainerCommandRequestProto requestProto, long entryIndex) {
- final WriteChunkRequestProto write = requestProto.getWriteChunk();
- String containerName = write.getPipeline().getContainerName();
- CompletableFuture<Message> future =
- createContainerFutureMap.get(containerName);
- CompletableFuture<Message> writeChunkFuture;
- if (future != null) {
- writeChunkFuture = future.thenApplyAsync(
- v -> runCommand(requestProto), writeChunkExecutor);
- } else {
- writeChunkFuture = CompletableFuture.supplyAsync(
- () -> runCommand(requestProto), writeChunkExecutor);
- }
- writeChunkFutureMap.put(entryIndex, writeChunkFuture);
- return writeChunkFuture;
- }
-
- private CompletableFuture<Message> handleCreateContainer(
- ContainerCommandRequestProto requestProto) {
- String containerName =
- requestProto.getCreateContainer().getContainerData().getName();
- createContainerFutureMap.
- computeIfAbsent(containerName, k -> new CompletableFuture<>());
- return CompletableFuture.completedFuture(() -> ByteString.EMPTY);
- }
-
- @Override
- public CompletableFuture<Message> writeStateMachineData(LogEntryProto entry) {
- try {
- final ContainerCommandRequestProto requestProto =
- getRequestProto(entry.getSmLogEntry().getStateMachineData());
- ContainerProtos.Type cmdType = requestProto.getCmdType();
- switch (cmdType) {
- case CreateContainer:
- return handleCreateContainer(requestProto);
- case WriteChunk:
- return handleWriteChunk(requestProto, entry.getIndex());
- default:
- throw new IllegalStateException("Cmd Type:" + cmdType
- + " should not have state machine data");
- }
- } catch (IOException e) {
- return completeExceptionally(e);
- }
- }
-
- @Override
- public CompletableFuture<Message> query(Message request) {
- try {
- final ContainerCommandRequestProto requestProto =
- getRequestProto(request.getContent());
- return CompletableFuture.completedFuture(runCommand(requestProto));
- } catch (IOException e) {
- return completeExceptionally(e);
- }
- }
-
- @Override
- public CompletableFuture<Message> applyTransaction(TransactionContext trx) {
- try {
- ContainerCommandRequestProto requestProto =
- getRequestProto(trx.getSMLogEntry().getData());
- ContainerProtos.Type cmdType = requestProto.getCmdType();
-
- if (cmdType == ContainerProtos.Type.WriteChunk) {
- WriteChunkRequestProto write = requestProto.getWriteChunk();
- // the data field has already been removed in start Transaction
- Preconditions.checkArgument(!write.hasData());
- CompletableFuture<Message> stateMachineFuture =
- writeChunkFutureMap.remove(trx.getLogEntry().getIndex());
- return stateMachineFuture
- .thenComposeAsync(v ->
- CompletableFuture.completedFuture(runCommand(requestProto)));
- } else {
- Message message = runCommand(requestProto);
- if (cmdType == ContainerProtos.Type.CreateContainer) {
- String containerName =
- requestProto.getCreateContainer().getContainerData().getName();
- createContainerFutureMap.remove(containerName).complete(message);
- }
- return CompletableFuture.completedFuture(message);
- }
- } catch (IOException e) {
- return completeExceptionally(e);
- }
- }
-
- private static <T> CompletableFuture<T> completeExceptionally(Exception e) {
- final CompletableFuture<T> future = new CompletableFuture<>();
- future.completeExceptionally(e);
- return future;
- }
-
- @Override
- public void close() throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
deleted file mode 100644
index d0ff094..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.transport.server
- .XceiverServerSpi;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.ratis.RaftConfigKeys;
-import org.apache.ratis.conf.RaftProperties;
-import org.apache.ratis.grpc.GrpcConfigKeys;
-import org.apache.ratis.netty.NettyConfigKeys;
-import org.apache.ratis.RatisHelper;
-import org.apache.ratis.rpc.RpcType;
-import org.apache.ratis.rpc.SupportedRpcType;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.RaftServerConfigKeys;
-import org.apache.ratis.util.SizeInBytes;
-import org.apache.ratis.util.TimeDuration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.net.SocketAddress;
-import java.util.Objects;
-import java.util.concurrent.ArrayBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-/**
- * Creates a ratis server endpoint that acts as the communication layer for
- * Ozone containers.
- */
-public final class XceiverServerRatis implements XceiverServerSpi {
- static final Logger LOG = LoggerFactory.getLogger(XceiverServerRatis.class);
- private final int port;
- private final RaftServer server;
- private ThreadPoolExecutor writeChunkExecutor;
-
- private XceiverServerRatis(DatanodeID id, int port, String storageDir,
- ContainerDispatcher dispatcher, Configuration conf) throws IOException {
-
- final String rpcType = conf.get(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_RPC_TYPE_DEFAULT);
- final RpcType rpc = SupportedRpcType.valueOfIgnoreCase(rpcType);
- final int raftSegmentSize = conf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_SIZE_DEFAULT);
- final int raftSegmentPreallocatedSize = conf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_SEGMENT_PREALLOCATED_SIZE_DEFAULT);
- final int maxChunkSize = OzoneConfigKeys.DFS_CONTAINER_CHUNK_MAX_SIZE;
- final int numWriteChunkThreads = conf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT);
-
- Objects.requireNonNull(id, "id == null");
- this.port = port;
- RaftProperties serverProperties = newRaftProperties(rpc, port,
- storageDir, maxChunkSize, raftSegmentSize, raftSegmentPreallocatedSize);
-
- writeChunkExecutor =
- new ThreadPoolExecutor(numWriteChunkThreads, numWriteChunkThreads,
- 100, TimeUnit.SECONDS,
- new ArrayBlockingQueue<>(1024),
- new ThreadPoolExecutor.CallerRunsPolicy());
- ContainerStateMachine stateMachine =
- new ContainerStateMachine(dispatcher, writeChunkExecutor);
- this.server = RaftServer.newBuilder()
- .setServerId(RatisHelper.toRaftPeerId(id))
- .setGroup(RatisHelper.emptyRaftGroup())
- .setProperties(serverProperties)
- .setStateMachine(stateMachine)
- .build();
- }
-
- private static RaftProperties newRaftProperties(
- RpcType rpc, int port, String storageDir, int scmChunkSize,
- int raftSegmentSize, int raftSegmentPreallocatedSize) {
- final RaftProperties properties = new RaftProperties();
- RaftServerConfigKeys.Log.Appender.setBatchEnabled(properties, true);
- RaftServerConfigKeys.Log.Appender.setBufferCapacity(properties,
- SizeInBytes.valueOf(raftSegmentPreallocatedSize));
- RaftServerConfigKeys.Log.setWriteBufferSize(properties,
- SizeInBytes.valueOf(scmChunkSize));
- RaftServerConfigKeys.Log.setPreallocatedSize(properties,
- SizeInBytes.valueOf(raftSegmentPreallocatedSize));
- RaftServerConfigKeys.Log.setSegmentSizeMax(properties,
- SizeInBytes.valueOf(raftSegmentSize));
- RaftServerConfigKeys.setStorageDir(properties, new File(storageDir));
- RaftConfigKeys.Rpc.setType(properties, rpc);
-
- RaftServerConfigKeys.Log.setMaxCachedSegmentNum(properties, 2);
- GrpcConfigKeys.setMessageSizeMax(properties,
- SizeInBytes.valueOf(scmChunkSize + raftSegmentPreallocatedSize));
- RaftServerConfigKeys.Rpc.setTimeoutMin(properties,
- TimeDuration.valueOf(800, TimeUnit.MILLISECONDS));
- RaftServerConfigKeys.Rpc.setTimeoutMax(properties,
- TimeDuration.valueOf(1000, TimeUnit.MILLISECONDS));
- if (rpc == SupportedRpcType.GRPC) {
- GrpcConfigKeys.Server.setPort(properties, port);
- } else if (rpc == SupportedRpcType.NETTY) {
- NettyConfigKeys.Server.setPort(properties, port);
- }
- return properties;
- }
-
- public static XceiverServerRatis newXceiverServerRatis(DatanodeID datanodeID,
- Configuration ozoneConf, ContainerDispatcher dispatcher)
- throws IOException {
- final String ratisDir = File.separator + "ratis";
- int localPort = ozoneConf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT_DEFAULT);
- String storageDir = ozoneConf.get(
- OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR);
-
- if (Strings.isNullOrEmpty(storageDir)) {
- storageDir = ozoneConf.get(OzoneConfigKeys
- .OZONE_METADATA_DIRS);
- Preconditions.checkNotNull(storageDir, "ozone.metadata.dirs " +
- "cannot be null, Please check your configs.");
- storageDir = storageDir.concat(ratisDir);
- LOG.warn("Storage directory for Ratis is not configured. Mapping Ratis " +
- "storage under {}. It is a good idea to map this to an SSD disk.",
- storageDir);
- }
-
- // Get an available port on current node and
- // use that as the container port
- if (ozoneConf.getBoolean(OzoneConfigKeys
- .DFS_CONTAINER_RATIS_IPC_RANDOM_PORT,
- OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT_DEFAULT)) {
- try (ServerSocket socket = new ServerSocket()) {
- socket.setReuseAddress(true);
- SocketAddress address = new InetSocketAddress(0);
- socket.bind(address);
- localPort = socket.getLocalPort();
- LOG.info("Found a free port for the server : {}", localPort);
- // If we have random local ports configured this means that it
- // probably running under MiniOzoneCluster. Ratis locks the storage
- // directories, so we need to pass different local directory for each
- // local instance. So we map ratis directories under datanode ID.
- storageDir =
- storageDir.concat(File.separator + datanodeID.getDatanodeUuid());
- } catch (IOException e) {
- LOG.error("Unable find a random free port for the server, "
- + "fallback to use default port {}", localPort, e);
- }
- }
- datanodeID.setRatisPort(localPort);
- return new XceiverServerRatis(datanodeID, localPort, storageDir,
- dispatcher, ozoneConf);
- }
-
- @Override
- public void start() throws IOException {
- LOG.info("Starting {} {} at port {}", getClass().getSimpleName(),
- server.getId(), getIPCPort());
- writeChunkExecutor.prestartAllCoreThreads();
- server.start();
- }
-
- @Override
- public void stop() {
- try {
- writeChunkExecutor.shutdown();
- server.close();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- @Override
- public int getIPCPort() {
- return port;
- }
-
- /**
- * Returns the Replication type supported by this end-point.
- *
- * @return enum -- {Stand_Alone, Ratis, Chained}
- */
- @Override
- public OzoneProtos.ReplicationType getServerType() {
- return OzoneProtos.ReplicationType.RATIS;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
deleted file mode 100644
index 8debfe0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.transport.server.ratis;
-
-/**
- * This package contains classes for the server implementation
- * using Apache Ratis
- */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
deleted file mode 100644
index 6ae45b6..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/ContainerCache.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.ozone.container.common.utils;
-
-import com.google.common.base.Preconditions;
-import org.apache.commons.collections.MapIterator;
-import org.apache.commons.collections.map.LRUMap;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.utils.MetadataStore;
-import org.apache.hadoop.utils.MetadataStoreBuilder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
-
-/**
- * container cache is a LRUMap that maintains the DB handles.
- */
-public final class ContainerCache extends LRUMap {
- private static final Logger LOG =
- LoggerFactory.getLogger(ContainerCache.class);
- private final Lock lock = new ReentrantLock();
- private static ContainerCache cache;
- private static final float LOAD_FACTOR = 0.75f;
- /**
- * Constructs a cache that holds DBHandle references.
- */
- private ContainerCache(int maxSize, float loadFactor, boolean
- scanUntilRemovable) {
- super(maxSize, loadFactor, scanUntilRemovable);
- }
-
- /**
- * Return a singleton instance of {@link ContainerCache}
- * that holds the DB handlers.
- *
- * @param conf - Configuration.
- * @return A instance of {@link ContainerCache}.
- */
- public synchronized static ContainerCache getInstance(Configuration conf) {
- if (cache == null) {
- int cacheSize = conf.getInt(OzoneConfigKeys.OZONE_CONTAINER_CACHE_SIZE,
- OzoneConfigKeys.OZONE_CONTAINER_CACHE_DEFAULT);
- cache = new ContainerCache(cacheSize, LOAD_FACTOR, true);
- }
- return cache;
- }
-
- /**
- * Closes a db instance.
- *
- * @param container - name of the container to be closed.
- * @param db - db instance to close.
- */
- private void closeDB(String container, MetadataStore db) {
- if (db != null) {
- try {
- db.close();
- } catch (IOException e) {
- LOG.error("Error closing DB. Container: " + container, e);
- }
- }
- }
-
- /**
- * Closes all the db instances and resets the cache.
- */
- public void shutdownCache() {
- lock.lock();
- try {
- // iterate the cache and close each db
- MapIterator iterator = cache.mapIterator();
- while (iterator.hasNext()) {
- iterator.next();
- MetadataStore db = (MetadataStore) iterator.getValue();
- closeDB(iterator.getKey().toString(), db);
- }
- // reset the cache
- cache.clear();
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * {@inheritDoc}
- */
- @Override
- protected boolean removeLRU(LinkEntry entry) {
- lock.lock();
- try {
- MetadataStore db = (MetadataStore) entry.getValue();
- closeDB(entry.getKey().toString(), db);
- } finally {
- lock.unlock();
- }
- return true;
- }
-
- /**
- * Returns a DB handle if available, create the handler otherwise.
- *
- * @param containerName - Name of the container.
- * @return MetadataStore.
- */
- public MetadataStore getDB(String containerName, String containerDBPath)
- throws IOException {
- Preconditions.checkNotNull(containerName);
- Preconditions.checkState(!containerName.isEmpty());
- lock.lock();
- try {
- MetadataStore db = (MetadataStore) this.get(containerName);
-
- if (db == null) {
- db = MetadataStoreBuilder.newBuilder()
- .setDbFile(new File(containerDBPath))
- .setCreateIfMissing(false)
- .build();
- this.put(containerName, db);
- }
- return db;
- } catch (Exception e) {
- LOG.error("Error opening DB. Container:{} ContainerPath:{}",
- containerName, containerDBPath, e);
- throw e;
- } finally {
- lock.unlock();
- }
- }
-
- /**
- * Remove a DB handler from cache.
- *
- * @param containerName - Name of the container.
- */
- public void removeDB(String containerName) {
- Preconditions.checkNotNull(containerName);
- Preconditions.checkState(!containerName.isEmpty());
- lock.lock();
- try {
- MetadataStore db = (MetadataStore)this.get(containerName);
- closeDB(containerName, db);
- this.remove(containerName);
- } finally {
- lock.unlock();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
deleted file mode 100644
index 08264f0..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/common/utils/package-info.java
+++ /dev/null
@@ -1,18 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.common.utils;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
deleted file mode 100644
index b143f22..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/OzoneContainer.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-
-package org.apache.hadoop.ozone.container.ozoneimpl;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.protocol.DatanodeID;
-import org.apache.hadoop.hdfs.server.datanode.StorageLocation;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.container.common.helpers.ContainerData;
-import org.apache.hadoop.ozone.container.common.impl.ChunkManagerImpl;
-import org.apache.hadoop.ozone.container.common.impl.ContainerManagerImpl;
-import org.apache.hadoop.ozone.container.common.impl.Dispatcher;
-import org.apache.hadoop.ozone.container.common.impl.KeyManagerImpl;
-import org.apache.hadoop.ozone.container.common.interfaces.ChunkManager;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerDispatcher;
-import org.apache.hadoop.ozone.container.common.interfaces.ContainerManager;
-import org.apache.hadoop.ozone.container.common.interfaces.KeyManager;
-import org.apache.hadoop.ozone.container.common.statemachine.background.BlockDeletingService;
-import org.apache.hadoop.ozone.container.common.transport.server.XceiverServer;
-import org.apache.hadoop.ozone.container.common.transport.server.ratis.XceiverServerRatis;
-
-import org.apache.hadoop.ozone.protocol.proto.OzoneProtos;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ContainerReportsRequestProto;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.ReportState;
-import org.apache.hadoop.ozone.protocol.proto.StorageContainerDatanodeProtocolProtos.SCMNodeReport;
-import org.apache.hadoop.ozone.container.common.transport.server.XceiverServerSpi;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY;
-import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_ROOT_PREFIX;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
-import static org.apache.hadoop.ozone.OzoneConfigKeys
- .OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT;
-import static org.apache.hadoop.ozone.OzoneConsts.INVALID_PORT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT;
-
-/**
- * Ozone main class sets up the network server and initializes the container
- * layer.
- */
-public class OzoneContainer {
- private static final Logger LOG =
- LoggerFactory.getLogger(OzoneContainer.class);
-
- private final Configuration ozoneConfig;
- private final ContainerDispatcher dispatcher;
- private final ContainerManager manager;
- private final XceiverServerSpi[] server;
- private final ChunkManager chunkManager;
- private final KeyManager keyManager;
- private final BlockDeletingService blockDeletingService;
-
- /**
- * Creates a network endpoint and enables Ozone container.
- *
- * @param ozoneConfig - Config
- * @throws IOException
- */
- public OzoneContainer(DatanodeID datanodeID, Configuration ozoneConfig) throws
- IOException {
- this.ozoneConfig = ozoneConfig;
- List<StorageLocation> locations = new LinkedList<>();
- String[] paths = ozoneConfig.getStrings(
- OzoneConfigKeys.OZONE_METADATA_DIRS);
- if (paths != null && paths.length > 0) {
- for (String p : paths) {
- locations.add(StorageLocation.parse(
- Paths.get(p).resolve(CONTAINER_ROOT_PREFIX).toString()));
- }
- } else {
- getDataDir(locations);
- }
-
- manager = new ContainerManagerImpl();
- manager.init(this.ozoneConfig, locations, datanodeID);
- this.chunkManager = new ChunkManagerImpl(manager);
- manager.setChunkManager(this.chunkManager);
-
- this.keyManager = new KeyManagerImpl(manager, ozoneConfig);
- manager.setKeyManager(this.keyManager);
-
- long svcInterval =
- ozoneConfig.getTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL,
- OZONE_BLOCK_DELETING_SERVICE_INTERVAL_DEFAULT, TimeUnit.MILLISECONDS);
- long serviceTimeout = ozoneConfig.getTimeDuration(
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT,
- OZONE_BLOCK_DELETING_SERVICE_TIMEOUT_DEFAULT, TimeUnit.MILLISECONDS);
- this.blockDeletingService = new BlockDeletingService(manager,
- svcInterval, serviceTimeout, ozoneConfig);
-
- this.dispatcher = new Dispatcher(manager, this.ozoneConfig);
-
- server = new XceiverServerSpi[]{
- new XceiverServer(this.ozoneConfig, this.dispatcher),
- XceiverServerRatis
- .newXceiverServerRatis(datanodeID, ozoneConfig, dispatcher)
- };
- }
-
- /**
- * Starts serving requests to ozone container.
- *
- * @throws IOException
- */
- public void start() throws IOException {
- for (XceiverServerSpi serverinstance : server) {
- serverinstance.start();
- }
- blockDeletingService.start();
- dispatcher.init();
- }
-
- /**
- * Stops the ozone container.
- * <p>
- * Shutdown logic is not very obvious from the following code. if you need to
- * modify the logic, please keep these comments in mind. Here is the shutdown
- * sequence.
- * <p>
- * 1. We shutdown the network ports.
- * <p>
- * 2. Now we need to wait for all requests in-flight to finish.
- * <p>
- * 3. The container manager lock is a read-write lock with "Fairness"
- * enabled.
- * <p>
- * 4. This means that the waiting threads are served in a "first-come-first
- * -served" manner. Please note that this applies to waiting threads only.
- * <p>
- * 5. Since write locks are exclusive, if we are waiting to get a lock it
- * implies that we are waiting for in-flight operations to complete.
- * <p>
- * 6. if there are other write operations waiting on the reader-writer lock,
- * fairness guarantees that they will proceed before the shutdown lock
- * request.
- * <p>
- * 7. Since all operations either take a reader or writer lock of container
- * manager, we are guaranteed that we are the last operation since we have
- * closed the network port, and we wait until close is successful.
- * <p>
- * 8. We take the writer lock and call shutdown on each of the managers in
- * reverse order. That is chunkManager, keyManager and containerManager is
- * shutdown.
- */
- public void stop() {
- LOG.info("Attempting to stop container services.");
- for(XceiverServerSpi serverinstance: server) {
- serverinstance.stop();
- }
- dispatcher.shutdown();
-
- try {
- this.manager.writeLock();
- this.chunkManager.shutdown();
- this.keyManager.shutdown();
- this.manager.shutdown();
- this.blockDeletingService.shutdown();
- LOG.info("container services shutdown complete.");
- } catch (IOException ex) {
- LOG.warn("container service shutdown error:", ex);
- } finally {
- this.manager.writeUnlock();
- }
- }
-
- /**
- * Returns a paths to data dirs.
- *
- * @param pathList - List of paths.
- * @throws IOException
- */
- private void getDataDir(List<StorageLocation> pathList) throws IOException {
- for (String dir : ozoneConfig.getStrings(DFS_DATANODE_DATA_DIR_KEY)) {
- StorageLocation location = StorageLocation.parse(dir);
- pathList.add(location);
- }
- }
-
- /**
- * Returns node report of container storage usage.
- */
- public SCMNodeReport getNodeReport() throws IOException {
- return this.manager.getNodeReport();
- }
-
- private int getPortbyType(OzoneProtos.ReplicationType replicationType) {
- for (XceiverServerSpi serverinstance : server) {
- if (serverinstance.getServerType() == replicationType) {
- return serverinstance.getIPCPort();
- }
- }
- return INVALID_PORT;
- }
-
- /**
- * Returns the container server IPC port.
- *
- * @return Container server IPC port.
- */
- public int getContainerServerPort() {
- return getPortbyType(OzoneProtos.ReplicationType.STAND_ALONE);
- }
-
- /**
- * Returns the Ratis container Server IPC port.
- *
- * @return Ratis port.
- */
- public int getRatisContainerServerPort() {
- return getPortbyType(OzoneProtos.ReplicationType.RATIS);
- }
-
- /**
- * Returns container report.
- * @return - container report.
- * @throws IOException
- */
- public ContainerReportsRequestProto getContainerReport() throws IOException {
- return this.manager.getContainerReport();
- }
-
-// TODO: remove getContainerReports
- /**
- * Returns the list of closed containers.
- * @return - List of closed containers.
- * @throws IOException
- */
- public List<ContainerData> getContainerReports() throws IOException {
- return this.manager.getContainerReports();
- }
-
- @VisibleForTesting
- public ContainerManager getContainerManager() {
- return this.manager;
- }
-
- /**
- * Get the container report state to send via HB to SCM.
- * @return the container report state.
- */
- public ReportState getContainerReportState() {
- return this.manager.getContainerReportState();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
deleted file mode 100644
index c99c038..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/container/ozoneimpl/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.ozone.container.ozoneimpl;
-/**
- Ozone main that calls into the container layer
-**/
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ce23d9ad/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
deleted file mode 100644
index 6c75691..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/ksm/BucketManager.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with this
- * work for additional information regarding copyright ownership. The ASF
- * licenses this file to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,WITHOUT
- * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
- * License for the specific language governing permissions and limitations under
- * the License.
- */
-package org.apache.hadoop.ozone.ksm;
-
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketArgs;
-import org.apache.hadoop.ozone.ksm.helpers.KsmBucketInfo;
-
-import java.io.IOException;
-import java.util.List;
-
-/**
- * BucketManager handles all the bucket level operations.
- */
-public interface BucketManager {
- /**
- * Creates a bucket.
- * @param bucketInfo - KsmBucketInfo for creating bucket.
- */
- void createBucket(KsmBucketInfo bucketInfo) throws IOException;
- /**
- * Returns Bucket Information.
- * @param volumeName - Name of the Volume.
- * @param bucketName - Name of the Bucket.
- */
- KsmBucketInfo getBucketInfo(String volumeName, String bucketName)
- throws IOException;
-
- /**
- * Sets bucket property from args.
- * @param args - BucketArgs.
- * @throws IOException
- */
- void setBucketProperty(KsmBucketArgs args) throws IOException;
-
- /**
- * Deletes an existing empty bucket from volume.
- * @param volumeName - Name of the volume.
- * @param bucketName - Name of the bucket.
- * @throws IOException
- */
- void deleteBucket(String volumeName, String bucketName) throws IOException;
-
- /**
- * Returns a list of buckets represented by {@link KsmBucketInfo}
- * in the given volume.
- *
- * @param volumeName
- * Required parameter volume name determines buckets in which volume
- * to return.
- * @param startBucket
- * Optional start bucket name parameter indicating where to start
- * the bucket listing from, this key is excluded from the result.
- * @param bucketPrefix
- * Optional start key parameter, restricting the response to buckets
- * that begin with the specified name.
- * @param maxNumOfBuckets
- * The maximum number of buckets to return. It ensures
- * the size of the result will not exceed this limit.
- * @return a list of buckets.
- * @throws IOException
- */
- List<KsmBucketInfo> listBuckets(String volumeName,
- String startBucket, String bucketPrefix, int maxNumOfBuckets)
- throws IOException;
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org